Apache DataFusion SQL Query Engine
Refactor avg & sum signatures away from user defined (#18769)
Jeffrey Vo3b390d782ea3
diff --git a/datafusion/functions-aggregate/src/average.rs b/datafusion/functions-aggregate/src/average.rsindex bec1734e2e..f4b3e598c3 100644--- a/datafusion/functions-aggregate/src/average.rs+++ b/datafusion/functions-aggregate/src/average.rs@@ -31,18 +31,15 @@ use arrow::datatypes::{DECIMAL256_MAX_SCALE, DECIMAL32_MAX_PRECISION, DECIMAL32_MAX_SCALE,DECIMAL64_MAX_PRECISION, DECIMAL64_MAX_SCALE,};-use datafusion_common::plan_err;-use datafusion_common::{- exec_err, not_impl_err, utils::take_function_args, Result, ScalarValue,-};+use datafusion_common::types::{logical_float64, NativeType};+use datafusion_common::{exec_err, not_impl_err, Result, ScalarValue};use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};use datafusion_expr::utils::format_state_name;-use datafusion_expr::Volatility::Immutable;use datafusion_expr::{- Accumulator, AggregateUDFImpl, Documentation, EmitTo, Expr, GroupsAccumulator,- ReversedUDAF, Signature,+ Accumulator, AggregateUDFImpl, Coercion, Documentation, EmitTo, Expr,+ GroupsAccumulator, ReversedUDAF, Signature, TypeSignature, TypeSignatureClass,+ Volatility,};-use datafusion_functions_aggregate_common::aggregate::avg_distinct::{DecimalDistinctAvgAccumulator, Float64DistinctAvgAccumulator,};@@ -50,7 +47,6 @@ use datafusion_functions_aggregate_common::aggregate::groups_accumulator::accumuuse datafusion_functions_aggregate_common::aggregate::groups_accumulator::nulls::{filtered_null_mask, set_nulls,};-use datafusion_functions_aggregate_common::utils::DecimalAverager;use datafusion_macros::user_doc;use log::debug;@@ -101,7 +97,24 @@ pub struct Avg {impl Avg {pub fn new() -> Self {Self {- signature: Signature::user_defined(Immutable),+ // Supported types smallint, int, bigint, real, double precision, decimal, or interval+ // Refer to https://www.postgresql.org/docs/8.2/functions-aggregate.html doc+ signature: Signature::one_of(+ vec![+ TypeSignature::Coercible(vec![Coercion::new_exact(+ TypeSignatureClass::Decimal,+ )]),+ TypeSignature::Coercible(vec![Coercion::new_exact(+ TypeSignatureClass::Duration,+ )]),+ TypeSignature::Coercible(vec![Coercion::new_implicit(+ TypeSignatureClass::Native(logical_float64()),+ vec![TypeSignatureClass::Integer, TypeSignatureClass::Float],+ NativeType::Float64,+ )]),+ ],+ Volatility::Immutable,+ ),aliases: vec![String::from("mean")],}}@@ -126,28 +139,6 @@ impl AggregateUDFImpl for Avg {&self.signature}- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {- let [args] = take_function_args(self.name(), arg_types)?;-- // Supported types smallint, int, bigint, real, double precision, decimal, or interval- // Refer to https://www.postgresql.org/docs/8.2/functions-aggregate.html doc- fn coerced_type(data_type: &DataType) -> Result<DataType> {- match &data_type {- DataType::Decimal32(p, s) => Ok(DataType::Decimal32(*p, *s)),- DataType::Decimal64(p, s) => Ok(DataType::Decimal64(*p, *s)),- DataType::Decimal128(p, s) => Ok(DataType::Decimal128(*p, *s)),- DataType::Decimal256(p, s) => Ok(DataType::Decimal256(*p, *s)),- d if d.is_numeric() => Ok(DataType::Float64),- DataType::Duration(time_unit) => Ok(DataType::Duration(*time_unit)),- DataType::Dictionary(_, v) => coerced_type(v.as_ref()),- _ => {- plan_err!("Avg does not support inputs of type {data_type}.")- }- }- }- Ok(vec![coerced_type(args)?])- }-fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {match &arg_types[0] {DataType::Decimal32(precision, scale) => {diff --git a/datafusion/functions-aggregate/src/sum.rs b/datafusion/functions-aggregate/src/sum.rsindex 958553d78c..d726b9fad1 100644--- a/datafusion/functions-aggregate/src/sum.rs+++ b/datafusion/functions-aggregate/src/sum.rs@@ -18,35 +18,31 @@//! Defines `SUM` and `SUM DISTINCT` aggregate accumulatorsuse ahash::RandomState;-use arrow::datatypes::DECIMAL32_MAX_PRECISION;-use arrow::datatypes::DECIMAL64_MAX_PRECISION;-use datafusion_expr::utils::AggregateOrderSensitivity;-use datafusion_expr::Expr;-use std::any::Any;-use std::mem::size_of_val;--use arrow::array::Array;-use arrow::array::ArrowNativeTypeOp;-use arrow::array::{ArrowNumericType, AsArray};-use arrow::datatypes::{ArrowNativeType, FieldRef};+use arrow::array::{Array, ArrayRef, ArrowNativeTypeOp, ArrowNumericType, AsArray};+use arrow::datatypes::Field;use arrow::datatypes::{- DataType, Decimal128Type, Decimal256Type, Decimal32Type, Decimal64Type, Float64Type,- Int64Type, UInt64Type, DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION,+ ArrowNativeType, DataType, Decimal128Type, Decimal256Type, Decimal32Type,+ Decimal64Type, FieldRef, Float64Type, Int64Type, UInt64Type,+ DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION, DECIMAL32_MAX_PRECISION,+ DECIMAL64_MAX_PRECISION,};-use arrow::{array::ArrayRef, datatypes::Field};-use datafusion_common::{- exec_err, not_impl_err, utils::take_function_args, HashMap, Result, ScalarValue,+use datafusion_common::types::{+ logical_float64, logical_int16, logical_int32, logical_int64, logical_int8,+ logical_uint16, logical_uint32, logical_uint64, logical_uint8, NativeType,};-use datafusion_expr::function::AccumulatorArgs;-use datafusion_expr::function::StateFieldsArgs;-use datafusion_expr::utils::format_state_name;+use datafusion_common::{exec_err, not_impl_err, HashMap, Result, ScalarValue};+use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};+use datafusion_expr::utils::{format_state_name, AggregateOrderSensitivity};use datafusion_expr::{- Accumulator, AggregateUDFImpl, Documentation, GroupsAccumulator, ReversedUDAF,- SetMonotonicity, Signature, Volatility,+ Accumulator, AggregateUDFImpl, Coercion, Documentation, Expr, GroupsAccumulator,+ ReversedUDAF, SetMonotonicity, Signature, TypeSignature, TypeSignatureClass,+ Volatility,};use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;use datafusion_functions_aggregate_common::aggregate::sum_distinct::DistinctSumAccumulator;use datafusion_macros::user_doc;+use std::any::Any;+use std::mem::size_of_val;make_udaf_expr_and_func!(Sum,@@ -130,7 +126,42 @@ pub struct Sum {impl Sum {pub fn new() -> Self {Self {- signature: Signature::user_defined(Volatility::Immutable),+ // Refer to https://www.postgresql.org/docs/8.2/functions-aggregate.html doc+ // smallint, int, bigint, real, double precision, decimal, or interval.+ signature: Signature::one_of(+ vec![+ TypeSignature::Coercible(vec![Coercion::new_exact(+ TypeSignatureClass::Decimal,+ )]),+ // Unsigned to u64+ TypeSignature::Coercible(vec![Coercion::new_implicit(+ TypeSignatureClass::Native(logical_uint64()),+ vec![+ TypeSignatureClass::Native(logical_uint8()),+ TypeSignatureClass::Native(logical_uint16()),+ TypeSignatureClass::Native(logical_uint32()),+ ],+ NativeType::UInt64,+ )]),+ // Signed to i64+ TypeSignature::Coercible(vec![Coercion::new_implicit(+ TypeSignatureClass::Native(logical_int64()),+ vec![+ TypeSignatureClass::Native(logical_int8()),+ TypeSignatureClass::Native(logical_int16()),+ TypeSignatureClass::Native(logical_int32()),+ ],+ NativeType::Int64,+ )]),+ // Floats to f64+ TypeSignature::Coercible(vec![Coercion::new_implicit(+ TypeSignatureClass::Native(logical_float64()),+ vec![TypeSignatureClass::Float],+ NativeType::Float64,+ )]),+ ],+ Volatility::Immutable,+ ),}}}@@ -154,57 +185,26 @@ impl AggregateUDFImpl for Sum {&self.signature}- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {- let [args] = take_function_args(self.name(), arg_types)?;-- // Refer to https://www.postgresql.org/docs/8.2/functions-aggregate.html doc- // smallint, int, bigint, real, double precision, decimal, or interval.-- fn coerced_type(data_type: &DataType) -> Result<DataType> {- match data_type {- DataType::Dictionary(_, v) => coerced_type(v),- // in the spark, the result type is DECIMAL(min(38,precision+10), s)- // ref: https://github.com/apache/spark/blob/fcf636d9eb8d645c24be3db2d599aba2d7e2955a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala#L66- DataType::Decimal32(_, _)- | DataType::Decimal64(_, _)- | DataType::Decimal128(_, _)- | DataType::Decimal256(_, _) => Ok(data_type.clone()),- dt if dt.is_signed_integer() => Ok(DataType::Int64),- dt if dt.is_unsigned_integer() => Ok(DataType::UInt64),- dt if dt.is_floating() => Ok(DataType::Float64),- _ => exec_err!("Sum not supported for {data_type}"),- }- }-- Ok(vec![coerced_type(args)?])- }-fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {match &arg_types[0] {DataType::Int64 => Ok(DataType::Int64),DataType::UInt64 => Ok(DataType::UInt64),DataType::Float64 => Ok(DataType::Float64),+ // In the spark, the result type is DECIMAL(min(38,precision+10), s)+ // ref: https://github.com/apache/spark/blob/fcf636d9eb8d645c24be3db2d599aba2d7e2955a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala#L66DataType::Decimal32(precision, scale) => {- // in the spark, the result type is DECIMAL(min(38,precision+10), s)- // ref: https://github.com/apache/spark/blob/fcf636d9eb8d645c24be3db2d599aba2d7e2955a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala#L66let new_precision = DECIMAL32_MAX_PRECISION.min(*precision + 10);Ok(DataType::Decimal32(new_precision, *scale))}DataType::Decimal64(precision, scale) => {- // in the spark, the result type is DECIMAL(min(38,precision+10), s)- // ref: https://github.com/apache/spark/blob/fcf636d9eb8d645c24be3db2d599aba2d7e2955a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala#L66let new_precision = DECIMAL64_MAX_PRECISION.min(*precision + 10);Ok(DataType::Decimal64(new_precision, *scale))}DataType::Decimal128(precision, scale) => {- // in the spark, the result type is DECIMAL(min(38,precision+10), s)- // ref: https://github.com/apache/spark/blob/fcf636d9eb8d645c24be3db2d599aba2d7e2955a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala#L66let new_precision = DECIMAL128_MAX_PRECISION.min(*precision + 10);Ok(DataType::Decimal128(new_precision, *scale))}DataType::Decimal256(precision, scale) => {- // in the spark, the result type is DECIMAL(min(38,precision+10), s)- // ref: https://github.com/apache/spark/blob/fcf636d9eb8d645c24be3db2d599aba2d7e2955a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala#L66let new_precision = DECIMAL256_MAX_PRECISION.min(*precision + 10);Ok(DataType::Decimal256(new_precision, *scale))}diff --git a/datafusion/spark/src/function/aggregate/avg.rs b/datafusion/spark/src/function/aggregate/avg.rsindex 65736815fe..4a7adc515b 100644--- a/datafusion/spark/src/function/aggregate/avg.rs+++ b/datafusion/spark/src/function/aggregate/avg.rs@@ -15,22 +15,21 @@// specific language governing permissions and limitations// under the License.-use arrow::array::ArrowNativeTypeOp;use arrow::array::{builder::PrimitiveBuilder,cast::AsArray,types::{Float64Type, Int64Type},- Array, ArrayRef, ArrowNumericType, Int64Array, PrimitiveArray,+ Array, ArrayRef, ArrowNativeTypeOp, ArrowNumericType, Int64Array, PrimitiveArray,};use arrow::compute::sum;use arrow::datatypes::{DataType, Field, FieldRef};-use datafusion_common::utils::take_function_args;-use datafusion_common::{not_impl_err, plan_err, Result, ScalarValue};+use datafusion_common::types::{logical_float64, NativeType};+use datafusion_common::{not_impl_err, Result, ScalarValue};use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};use datafusion_expr::utils::format_state_name;-use datafusion_expr::Volatility::Immutable;use datafusion_expr::{- Accumulator, AggregateUDFImpl, EmitTo, GroupsAccumulator, ReversedUDAF, Signature,+ Accumulator, AggregateUDFImpl, Coercion, EmitTo, GroupsAccumulator, ReversedUDAF,+ Signature, TypeSignatureClass, Volatility,};use std::{any::Any, sync::Arc};@@ -56,7 +55,14 @@ impl SparkAvg {/// Implement AVG aggregate functionpub fn new() -> Self {Self {- signature: Signature::user_defined(Immutable),+ signature: Signature::coercible(+ vec![Coercion::new_implicit(+ TypeSignatureClass::Native(logical_float64()),+ vec![TypeSignatureClass::Numeric],+ NativeType::Float64,+ )],+ Volatility::Immutable,+ ),}}}@@ -66,21 +72,6 @@ impl AggregateUDFImpl for SparkAvg {self}- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {- let [args] = take_function_args(self.name(), arg_types)?;-- fn coerced_type(data_type: &DataType) -> Result<DataType> {- match &data_type {- d if d.is_numeric() => Ok(DataType::Float64),- DataType::Dictionary(_, v) => coerced_type(v.as_ref()),- _ => {- plan_err!("Avg does not support inputs of type {data_type}.")- }- }- }- Ok(vec![coerced_type(args)?])- }-fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {Ok(DataType::Float64)}