Skip to content

Apache DataFusion SQL Query Engine

fix: Address avg return type overflow for decimals (#24371)

Adam Gutglick9ef3d47225b6
diff --git a/datafusion/functions-aggregate-common/src/aggregate/avg_distinct/decimal.rs b/datafusion/functions-aggregate-common/src/aggregate/avg_distinct/decimal.rs
index d9e06d3952..62db8622c5 100644
--- a/datafusion/functions-aggregate-common/src/aggregate/avg_distinct/decimal.rs
+++ b/datafusion/functions-aggregate-common/src/aggregate/avg_distinct/decimal.rs
@@ -32,22 +32,28 @@ use crate::utils::DecimalAverager;
/// Generic implementation of `AVG DISTINCT` for Decimal types.
/// Handles both all Arrow decimal types (32, 64, 128 and 256 bits).
///
-/// The distinct values are stored in the input type `I`; only the intermediate
-/// sum is computed in the (never narrower) sum type `S` so it cannot overflow
-/// `I`'s native type.
+/// The distinct values are stored in the input type `I`, the result is returned
+/// as the same or wider type `O`, and the intermediate sum is computed in the
+/// (never narrower) sum type `S` so it cannot overflow `I`'s native type.
#[derive(Debug)]
pub struct DecimalDistinctAvgAccumulator<
I: DecimalType + Debug,
S: DecimalType + Debug = I,
+ O: DecimalType + Debug = I,
> {
sum_accumulator: DistinctSumAccumulator<I>,
sum_scale: i8,
target_precision: u8,
target_scale: i8,
- _sum_type: PhantomData<S>,
+ _types: PhantomData<(S, O)>,
}
-impl<I: DecimalType + Debug, S: DecimalType + Debug> DecimalDistinctAvgAccumulator<I, S> {
+impl<I, S, O> DecimalDistinctAvgAccumulator<I, S, O>
+where
+ I: DecimalType + Debug,
+ S: DecimalType + Debug,
+ O: DecimalType + Debug,
+{
pub fn with_decimal_params(
sum_scale: i8,
target_precision: u8,
@@ -60,7 +66,7 @@ impl<I: DecimalType + Debug, S: DecimalType + Debug> DecimalDistinctAvgAccumulat
sum_scale,
target_precision,
target_scale,
- _sum_type: PhantomData,
+ _types: PhantomData,
}
}
}
@@ -78,12 +84,14 @@ where
sum.add_wrapping(value.into())
}
-impl<I, S> Accumulator for DecimalDistinctAvgAccumulator<I, S>
+impl<I, S, O> Accumulator for DecimalDistinctAvgAccumulator<I, S, O>
where
I: DecimalType + ArrowNumericType + Debug,
S: DecimalType + ArrowNumericType + Debug,
+ O: DecimalType + ArrowNumericType + Debug,
I::Native: Into<S::Native> + DecimalCast,
S::Native: DecimalCast,
+ O::Native: DecimalCast,
{
fn state(&mut self) -> Result<Vec<ScalarValue>> {
self.sum_accumulator.state()
@@ -98,10 +106,10 @@ where
}
fn evaluate(&mut self) -> Result<ScalarValue> {
- let out_type = I::TYPE_CONSTRUCTOR(self.target_precision, self.target_scale);
+ let out_type = O::TYPE_CONSTRUCTOR(self.target_precision, self.target_scale);
let count = self.sum_accumulator.distinct_count();
if count == 0 {
- return ScalarValue::new_primitive::<I>(None, &out_type);
+ return ScalarValue::new_primitive::<O>(None, &out_type);
}
// Sum the distinct input values in the wider `S` so the total cannot
@@ -123,18 +131,17 @@ where
self.target_precision,
self.target_scale,
)?;
- // Narrowing the average back to the (never wider) output type cannot
- // fail in practice: `DecimalAverager::avg` validates the average
- // against the output precision, whose bound fits the output's native
- // type by construction
+ // Converting the average to the output type cannot fail in practice:
+ // `DecimalAverager::avg` validates it against the output precision,
+ // whose bound fits the output's native type by construction.
let avg =
- I::Native::from_decimal(averager.avg(sum, count)?).ok_or_else(|| {
+ O::Native::from_decimal(averager.avg(sum, count)?).ok_or_else(|| {
exec_datafusion_err!(
"Arithmetic overflow in avg: the computed average does not fit \
- the output type"
+ the output type"
)
})?;
- ScalarValue::new_primitive::<I>(Some(avg), &out_type)
+ ScalarValue::new_primitive::<O>(Some(avg), &out_type)
}
fn size(&self) -> usize {
diff --git a/datafusion/functions-aggregate/src/average.rs b/datafusion/functions-aggregate/src/average.rs
index 8601d93c11..75bcec491d 100644
--- a/datafusion/functions-aggregate/src/average.rs
+++ b/datafusion/functions-aggregate/src/average.rs
@@ -24,9 +24,8 @@ use arrow::array::{
use arrow::compute::{DecimalCast, sum};
use arrow::datatypes::{
- ArrowNativeType, DECIMAL32_MAX_PRECISION, DECIMAL32_MAX_SCALE,
- DECIMAL64_MAX_PRECISION, DECIMAL64_MAX_SCALE, DECIMAL128_MAX_PRECISION,
- DECIMAL128_MAX_SCALE, DECIMAL256_MAX_PRECISION, DECIMAL256_MAX_SCALE, DataType,
+ ArrowNativeType, DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION,
+ DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION, DECIMAL256_MAX_SCALE, DataType,
Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, DecimalType,
DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType,
DurationSecondType, Field, FieldRef, Float64Type, TimeUnit, UInt64Type,
@@ -174,37 +173,99 @@ fn avg_sum_data_type(data_type: &DataType) -> DataType {
}
}
-/// Instantiates `$builder::<Input, Sum>` for every decimal pair that
-/// [`avg_sum_data_type`] can produce.
+/// Instantiates `$builder::<Input, Sum, Output>` for every combination
+/// [`avg_sum_data_type`] and [`avg_decimal_return_type`] can produce.
macro_rules! decimal_avg_dispatch {
- ($input:expr, $sum:expr, $builder:ident, $($arg:expr),*) => {
- match ($input, $sum) {
- (DataType::Decimal32(..), DataType::Decimal64(..)) => {
- $builder::<Decimal32Type, Decimal64Type>($($arg),*)
- }
- (DataType::Decimal32(..), DataType::Decimal128(..)) => {
- $builder::<Decimal32Type, Decimal128Type>($($arg),*)
- }
- (DataType::Decimal64(..), DataType::Decimal64(..)) => {
- $builder::<Decimal64Type, Decimal64Type>($($arg),*)
- }
- (DataType::Decimal64(..), DataType::Decimal128(..)) => {
- $builder::<Decimal64Type, Decimal128Type>($($arg),*)
- }
- (DataType::Decimal128(..), DataType::Decimal128(..)) => {
- $builder::<Decimal128Type, Decimal128Type>($($arg),*)
- }
- (DataType::Decimal128(..), DataType::Decimal256(..)) => {
- $builder::<Decimal128Type, Decimal256Type>($($arg),*)
- }
- (DataType::Decimal256(..), DataType::Decimal256(..)) => {
- $builder::<Decimal256Type, Decimal256Type>($($arg),*)
- }
- (input, sum) => {
- internal_err!("avg cannot accumulate {input} as {sum}")
+ ($input:expr, $sum:expr, $output:expr, $builder:ident) => {{
+ let input = $input;
+ let sum = $sum;
+ let output = $output;
+ match (input, sum, output) {
+ (
+ DataType::Decimal32(..),
+ DataType::Decimal64(..),
+ DataType::Decimal32(..),
+ ) => $builder::<Decimal32Type, Decimal64Type, Decimal32Type>(sum, output),
+ (
+ DataType::Decimal32(..),
+ DataType::Decimal128(..),
+ DataType::Decimal64(..),
+ ) => $builder::<Decimal32Type, Decimal128Type, Decimal64Type>(sum, output),
+ (
+ DataType::Decimal64(..),
+ DataType::Decimal64(..),
+ DataType::Decimal64(..),
+ ) => $builder::<Decimal64Type, Decimal64Type, Decimal64Type>(sum, output),
+ (
+ DataType::Decimal64(..),
+ DataType::Decimal128(..),
+ DataType::Decimal64(..),
+ ) => $builder::<Decimal64Type, Decimal128Type, Decimal64Type>(sum, output),
+ (
+ DataType::Decimal64(..),
+ DataType::Decimal128(..),
+ DataType::Decimal128(..),
+ ) => $builder::<Decimal64Type, Decimal128Type, Decimal128Type>(sum, output),
+ (
+ DataType::Decimal128(..),
+ DataType::Decimal128(..),
+ DataType::Decimal128(..),
+ ) => $builder::<Decimal128Type, Decimal128Type, Decimal128Type>(sum, output),
+ (
+ DataType::Decimal128(..),
+ DataType::Decimal256(..),
+ DataType::Decimal128(..),
+ ) => $builder::<Decimal128Type, Decimal256Type, Decimal128Type>(sum, output),
+ (
+ DataType::Decimal128(..),
+ DataType::Decimal256(..),
+ DataType::Decimal256(..),
+ ) => $builder::<Decimal128Type, Decimal256Type, Decimal256Type>(sum, output),
+ (
+ DataType::Decimal256(..),
+ DataType::Decimal256(..),
+ DataType::Decimal256(..),
+ ) => $builder::<Decimal256Type, Decimal256Type, Decimal256Type>(sum, output),
+ (input, sum, output) => {
+ internal_err!(
+ "avg cannot accumulate {input} as {sum} and return {output}"
+ )
}
}
- };
+ }};
+}
+
+/// Returns `Decimal(precision + 4, scale + 4)`, widening the Arrow decimal type
+/// when necessary and never narrowing it. Precision and scale are capped at the
+/// `Decimal256` limits. This follows [Spark's average type derivation] while
+/// accounting for Arrow's physical decimal types.
+///
+/// [Spark's average type derivation]: https://github.com/apache/spark/blob/fcf636d9eb8d645c24be3db2d599aba2d7e2955a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala#L66
+fn avg_decimal_return_type(data_type: &DataType) -> Result<DataType> {
+ let (precision, scale) = decimal_parts(data_type)?;
+ let precision = DECIMAL256_MAX_PRECISION.min(precision.saturating_add(4));
+ let scale = DECIMAL256_MAX_SCALE.min(scale.saturating_add(4));
+
+ Ok(match data_type {
+ DataType::Decimal32(..) if precision <= DECIMAL32_MAX_PRECISION => {
+ DataType::Decimal32(precision, scale)
+ }
+ DataType::Decimal32(..) | DataType::Decimal64(..)
+ if precision <= DECIMAL64_MAX_PRECISION =>
+ {
+ DataType::Decimal64(precision, scale)
+ }
+ DataType::Decimal32(..) | DataType::Decimal64(..) | DataType::Decimal128(..)
+ if precision <= DECIMAL128_MAX_PRECISION =>
+ {
+ DataType::Decimal128(precision, scale)
+ }
+ DataType::Decimal32(..)
+ | DataType::Decimal64(..)
+ | DataType::Decimal128(..)
+ | DataType::Decimal256(..) => DataType::Decimal256(precision, scale),
+ _ => unreachable!("decimal_parts accepted a non-decimal type"),
+ })
}
impl AggregateUDFImpl for Avg {
@@ -218,34 +279,10 @@ impl AggregateUDFImpl for Avg {
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
match &arg_types[0] {
- DataType::Decimal32(precision, scale) => {
- // In the spark, the result type is DECIMAL(min(38,precision+4), min(38,scale+4)).
- // Ref: https://github.com/apache/spark/blob/fcf636d9eb8d645c24be3db2d599aba2d7e2955a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala#L66
- let new_precision = DECIMAL32_MAX_PRECISION.min(*precision + 4);
- let new_scale = DECIMAL32_MAX_SCALE.min(*scale + 4);
- Ok(DataType::Decimal32(new_precision, new_scale))
- }
- DataType::Decimal64(precision, scale) => {
- // In the spark, the result type is DECIMAL(min(38,precision+4), min(38,scale+4)).
- // Ref: https://github.com/apache/spark/blob/fcf636d9eb8d645c24be3db2d599aba2d7e2955a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala#L66
- let new_precision = DECIMAL64_MAX_PRECISION.min(*precision + 4);
- let new_scale = DECIMAL64_MAX_SCALE.min(*scale + 4);
- Ok(DataType::Decimal64(new_precision, new_scale))
- }
- DataType::Decimal128(precision, scale) => {
- // In the spark, the result type is DECIMAL(min(38,precision+4), min(38,scale+4)).
- // Ref: https://github.com/apache/spark/blob/fcf636d9eb8d645c24be3db2d599aba2d7e2955a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala#L66
- let new_precision = DECIMAL128_MAX_PRECISION.min(*precision + 4);
- let new_scale = DECIMAL128_MAX_SCALE.min(*scale + 4);
- Ok(DataType::Decimal128(new_precision, new_scale))
- }
- DataType::Decimal256(precision, scale) => {
- // In the spark, the result type is DECIMAL(min(38,precision+4), min(38,scale+4)).
- // Ref: https://github.com/apache/spark/blob/fcf636d9eb8d645c24be3db2d599aba2d7e2955a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala#L66
- let new_precision = DECIMAL256_MAX_PRECISION.min(*precision + 4);
- let new_scale = DECIMAL256_MAX_SCALE.min(*scale + 4);
- Ok(DataType::Decimal256(new_precision, new_scale))
- }
+ data_type @ (DataType::Decimal32(..)
+ | DataType::Decimal64(..)
+ | DataType::Decimal128(..)
+ | DataType::Decimal256(..)) => avg_decimal_return_type(data_type),
DataType::Duration(time_unit) => Ok(DataType::Duration(*time_unit)),
_ => Ok(DataType::Float64),
}
@@ -261,17 +298,16 @@ impl AggregateUDFImpl for Avg {
// Numeric types are converted to Float64 via `coerce_avg_type` during logical plan creation
(Float64, _) => Ok(Box::new(Float64DistinctAvgAccumulator::default())),
- (Decimal32(..), Decimal32(..))
- | (Decimal64(..), Decimal64(..))
- | (Decimal128(..), Decimal128(..))
- | (Decimal256(..), Decimal256(..)) => {
+ (
+ Decimal32(..) | Decimal64(..) | Decimal128(..) | Decimal256(..),
+ Decimal32(..) | Decimal64(..) | Decimal128(..) | Decimal256(..),
+ ) => {
let sum_data_type = avg_sum_data_type(data_type);
decimal_avg_dispatch!(
data_type,
&sum_data_type,
- decimal_distinct_avg_accumulator,
- &sum_data_type,
- acc_args.return_type()
+ acc_args.return_type(),
+ decimal_distinct_avg_accumulator
)
}
@@ -284,17 +320,16 @@ impl AggregateUDFImpl for Avg {
} else {
match (&data_type, acc_args.return_type()) {
(Float64, Float64) => Ok(Box::<AvgAccumulator>::default()),
- (Decimal32(..), Decimal32(..))
- | (Decimal64(..), Decimal64(..))
- | (Decimal128(..), Decimal128(..))
- | (Decimal256(..), Decimal256(..)) => {
+ (
+ Decimal32(..) | Decimal64(..) | Decimal128(..) | Decimal256(..),
+ Decimal32(..) | Decimal64(..) | Decimal128(..) | Decimal256(..),
+ ) => {
let sum_data_type = avg_sum_data_type(data_type);
decimal_avg_dispatch!(
data_type,
&sum_data_type,
- decimal_avg_accumulator,
- sum_data_type.clone(),
- acc_args.return_type().clone()
+ acc_args.return_type(),
+ decimal_avg_accumulator
)
}
@@ -388,17 +423,16 @@ impl AggregateUDFImpl for Avg {
|sum: f64, count: u64| Ok(sum / count as f64),
)))
}
- (Decimal32(..), Decimal32(..))
- | (Decimal64(..), Decimal64(..))
- | (Decimal128(..), Decimal128(..))
- | (Decimal256(..), Decimal256(..)) => {
+ (
+ Decimal32(..) | Decimal64(..) | Decimal128(..) | Decimal256(..),
+ Decimal32(..) | Decimal64(..) | Decimal128(..) | Decimal256(..),
+ ) => {
let sum_data_type = avg_sum_data_type(data_type);
decimal_avg_dispatch!(
data_type,
&sum_data_type,
- decimal_avg_groups_accumulator,
- &sum_data_type,
- args.return_field.data_type()
+ args.return_field.data_type(),
+ decimal_avg_groups_accumulator
)
}
@@ -473,15 +507,15 @@ fn decimal_parts(data_type: &DataType) -> Result<(u8, i8)> {
}
}
-fn decimal_avg_fn<I, S>(
+fn decimal_avg_fn<O, S>(
sum_scale: i8,
target_precision: u8,
target_scale: i8,
-) -> Result<impl Fn(S::Native, u64) -> Result<I::Native> + Send + Sync + 'static>
+) -> Result<impl Fn(S::Native, u64) -> Result<O::Native> + Send + Sync + 'static>
where
- I: DecimalType,
+ O: DecimalType,
S: DecimalType,
- I::Native: DecimalCast,
+ O::Native: DecimalCast,
S::Native: DecimalCast,
{
let decimal_averager =
@@ -496,11 +530,10 @@ where
);
};
- // Narrowing the average back to the (never wider) output type cannot
- // fail in practice: `DecimalAverager::avg` validates the average
- // against the output precision, whose bound fits the output's native
- // type by construction
- I::Native::from_decimal(decimal_averager.avg(sum, count)?).ok_or_else(|| {
+ // Converting the average to the output type cannot fail in practice:
+ // `DecimalAverager::avg` validates it against the output precision,
+ // whose bound fits the output's native type by construction.
+ O::Native::from_decimal(decimal_averager.avg(sum, count)?).ok_or_else(|| {
exec_datafusion_err!(
"Arithmetic overflow in avg: the computed average does not fit \
the output type"
@@ -509,42 +542,46 @@ where
})
}
-fn decimal_avg_accumulator<I, S>(
- sum_data_type: DataType,
- return_data_type: DataType,
+fn decimal_avg_accumulator<I, S, O>(
+ sum_data_type: &DataType,
+ return_data_type: &DataType,
) -> Result<Box<dyn Accumulator>>
where
I: DecimalType + ArrowNumericType + Debug + Send + Sync,
+ O: DecimalType + ArrowNumericType + Debug + Send + Sync,
S: DecimalType + ArrowNumericType + Debug + Send + Sync,
I::Native: Into<S::Native> + DecimalCast,
+ O::Native: DecimalCast,
S::Native: DecimalCast,
{
- let (_, sum_scale) = decimal_parts(&sum_data_type)?;
- let (target_precision, target_scale) = decimal_parts(&return_data_type)?;
- let avg_fn = decimal_avg_fn::<I, S>(sum_scale, target_precision, target_scale)?;
+ let (_, sum_scale) = decimal_parts(sum_data_type)?;
+ let (target_precision, target_scale) = decimal_parts(return_data_type)?;
+ let avg_fn = decimal_avg_fn::<O, S>(sum_scale, target_precision, target_scale)?;
- Ok(Box::new(DecimalAvgAccumulator::<I, S, _>::new(
- sum_data_type,
- return_data_type,
+ Ok(Box::new(DecimalAvgAccumulator::<I, S, _, O>::new(
+ sum_data_type.clone(),
+ return_data_type.clone(),
avg_fn,
)))
}
-fn decimal_distinct_avg_accumulator<I, S>(
+fn decimal_distinct_avg_accumulator<I, S, O>(
sum_data_type: &DataType,
return_data_type: &DataType,
) -> Result<Box<dyn Accumulator>>
where
I: DecimalType + ArrowNumericType + Debug + Send + Sync,
+ O: DecimalType + ArrowNumericType + Debug + Send + Sync,
S: DecimalType + ArrowNumericType + Debug + Send + Sync,
I::Native: Into<S::Native> + DecimalCast,
+ O::Native: DecimalCast,
S::Native: DecimalCast,
{
let (_, sum_scale) = decimal_parts(sum_data_type)?;
let (target_precision, target_scale) = decimal_parts(return_data_type)?;
Ok(Box::new(
- DecimalDistinctAvgAccumulator::<I, S>::with_decimal_params(
+ DecimalDistinctAvgAccumulator::<I, S, O>::with_decimal_params(
sum_scale,
target_precision,
target_scale,
@@ -552,21 +589,23 @@ where
))
}
-fn decimal_avg_groups_accumulator<I, S>(
+fn decimal_avg_groups_accumulator<I, S, O>(
sum_data_type: &DataType,
return_data_type: &DataType,
) -> Result<Box<dyn GroupsAccumulator>>
where
I: DecimalType + ArrowNumericType + Debug + Send + Sync,
+ O: DecimalType + ArrowNumericType + Debug + Send + Sync,
S: DecimalType + ArrowNumericType + Debug + Send + Sync,
I::Native: Into<S::Native> + DecimalCast,
+ O::Native: DecimalCast,
S::Native: DecimalCast,
{
let (_, sum_scale) = decimal_parts(sum_data_type)?;
let (target_precision, target_scale) = decimal_parts(return_data_type)?;
- let avg_fn = decimal_avg_fn::<I, S>(sum_scale, target_precision, target_scale)?;
+ let avg_fn = decimal_avg_fn::<O, S>(sum_scale, target_precision, target_scale)?;
- Ok(Box::new(AvgGroupsAccumulator::<I, _, S>::new(
+ Ok(Box::new(AvgGroupsAccumulator::<I, _, S, O>::new(
sum_data_type,
return_data_type,
avg_fn,
@@ -642,29 +681,32 @@ impl Accumulator for AvgAccumulator {
/// An accumulator to compute the average for decimals.
///
-/// `I` is the input (and output) decimal type. `S` is the type used to accumulate
-/// the sum, chosen by [`avg_sum_data_type`] so the running total does not overflow.
-struct DecimalAvgAccumulator<I, S, F>
+/// `I` is the input decimal type, `O` is the output decimal type, and `S` is the
+/// type used to accumulate the sum, chosen by [`avg_sum_data_type`] so the
+/// running total does not overflow.
+struct DecimalAvgAccumulator<I, S, F, O = I>
where
I: DecimalType + ArrowNumericType + Debug,
+ O: DecimalType + ArrowNumericType + Debug,
S: DecimalType + ArrowNumericType + Debug,
I::Native: Into<S::Native>,
- F: Fn(S::Native, u64) -> Result<I::Native>,
+ F: Fn(S::Native, u64) -> Result<O::Native>,
{
sum: Option<S::Native>,
count: u64,
sum_data_type: DataType,
return_data_type: DataType,
avg_fn: F,
- _phantom: PhantomData<I>,
+ _phantom: PhantomData<(I, O)>,
}
-impl<I, S, F> Debug for DecimalAvgAccumulator<I, S, F>
+impl<I, S, F, O> Debug for DecimalAvgAccumulator<I, S, F, O>
where
I: DecimalType + ArrowNumericType + Debug,
+ O: DecimalType + ArrowNumericType + Debug,
S: DecimalType + ArrowNumericType + Debug,
I::Native: Into<S::Native>,
- F: Fn(S::Native, u64) -> Result<I::Native>,
+ F: Fn(S::Native, u64) -> Result<O::Native>,
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DecimalAvgAccumulator")
@@ -676,12 +718,13 @@ where
}
}
-impl<I, S, F> DecimalAvgAccumulator<I, S, F>
+impl<I, S, F, O> DecimalAvgAccumulator<I, S, F, O>
where
I: DecimalType + ArrowNumericType + Debug,
+ O: DecimalType + ArrowNumericType + Debug,
S: DecimalType + ArrowNumericType + Debug,
I::Native: Into<S::Native>,
- F: Fn(S::Native, u64) -> Result<I::Native>,
+ F: Fn(S::Native, u64) -> Result<O::Native>,
{
fn new(sum_data_type: DataType, return_data_type: DataType, avg_fn: F) -> Self {
Self {
@@ -748,12 +791,13 @@ where
Some(sum)
}
-impl<I, S, F> Accumulator for DecimalAvgAccumulator<I, S, F>
+impl<I, S, F, O> Accumulator for DecimalAvgAccumulator<I, S, F, O>
where
I: DecimalType + ArrowNumericType + Debug,
+ O: DecimalType + ArrowNumericType + Debug,
S: DecimalType + ArrowNumericType + Debug,
I::Native: Into<S::Native>,
- F: Fn(S::Native, u64) -> Result<I::Native> + Send + Sync + 'static,
+ F: Fn(S::Native, u64) -> Result<O::Native> + Send + Sync + 'static,
{
fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
let values = values[0].as_primitive::<I>();
@@ -776,7 +820,7 @@ where
self.sum.map(|v| (self.avg_fn)(v, self.count)).transpose()?
};
- ScalarValue::new_primitive::<I>(v, &self.return_data_type)
+ ScalarValue::new_primitive::<O>(v, &self.return_data_type)
}
fn size(&self) -> usize {
@@ -928,20 +972,21 @@ impl Accumulator for DurationAvgAccumulator {
/// F: Function that calculates the average value from a sum of
/// S::Native and a total count
///
-/// `I` is the input (and output) type. `S` is a possibly wider type used to
-/// accumulate the sum so it does not overflow.
+/// `I` is the input type, `O` is the output type, and `S` is a possibly wider
+/// type used to accumulate the sum so it does not overflow.
#[derive(Debug)]
-struct AvgGroupsAccumulator<I, F, S = I>
+struct AvgGroupsAccumulator<I, F, S = I, O = I>
where
I: ArrowNumericType + Send,
+ O: ArrowNumericType + Send,
S: ArrowNumericType + Send,
I::Native: Into<S::Native>,
- F: Fn(S::Native, u64) -> Result<I::Native> + Send + 'static,
+ F: Fn(S::Native, u64) -> Result<O::Native> + Send + 'static,
{
/// The type of the internal sum
sum_data_type: DataType,
- /// The type of the returned sum
+ /// The type of the returned average
return_data_type: DataType,
/// Count per group (use u64 to make UInt64Array)
@@ -956,15 +1001,16 @@ where
/// Function that computes the final average (value / count)
avg_fn: F,
- _phantom: PhantomData<I>,
+ _phantom: PhantomData<(I, O)>,
}
-impl<I, F, S> AvgGroupsAccumulator<I, F, S>
+impl<I, F, S, O> AvgGroupsAccumulator<I, F, S, O>
where
I: ArrowNumericType + Send,
+ O: ArrowNumericType + Send,
S: ArrowNumericType + Send,
I::Native: Into<S::Native>,
- F: Fn(S::Native, u64) -> Result<I::Native> + Send + 'static,
+ F: Fn(S::Native, u64) -> Result<O::Native> + Send + 'static,
{
pub fn new(sum_data_type: &DataType, return_data_type: &DataType, avg_fn: F) -> Self {
debug!(
@@ -984,12 +1030,13 @@ where
}
}
-impl<I, F, S> GroupsAccumulator for AvgGroupsAccumulator<I, F, S>
+impl<I, F, S, O> GroupsAccumulator for AvgGroupsAccumulator<I, F, S, O>
where
I: ArrowNumericType + Send,
+ O: ArrowNumericType + Send,
S: ArrowNumericType + Send,
I::Native: Into<S::Native>,
- F: Fn(S::Native, u64) -> Result<I::Native> + Send + 'static,
+ F: Fn(S::Native, u64) -> Result<O::Native> + Send + 'static,
{
fn update_batch(
&mut self,
@@ -1034,10 +1081,10 @@ where
// don't evaluate averages with null inputs to avoid errors on null values
- let array: PrimitiveArray<I> = if let Some(nulls) = &nulls
+ let array: PrimitiveArray<O> = if let Some(nulls) = &nulls
&& nulls.null_count() > 0
{
- let mut builder = PrimitiveBuilder::<I>::with_capacity(nulls.len())
+ let mut builder = PrimitiveBuilder::<O>::with_capacity(nulls.len())
.with_data_type(self.return_data_type.clone());
let iter = sums.into_iter().zip(counts).zip(nulls.iter());
@@ -1050,7 +1097,7 @@ where
}
builder.finish()
} else {
- let averages: Vec<I::Native> = sums
+ let averages: Vec<O::Native> = sums
.into_iter()
.zip(counts)
.map(|(sum, count)| (self.avg_fn)(sum, count))
@@ -1189,6 +1236,15 @@ mod tests {
input_type: &DataType,
return_type: &DataType,
f: impl FnOnce(AccumulatorArgs) -> R,
+ ) -> R {
+ with_avg_args_distinct(input_type, return_type, false, f)
+ }
+
+ fn with_avg_args_distinct<R>(
+ input_type: &DataType,
+ return_type: &DataType,
+ is_distinct: bool,
+ f: impl FnOnce(AccumulatorArgs) -> R,
) -> R {
let schema = Schema::empty();
let expr_field = Arc::new(Field::new("a", input_type.clone(), true));
@@ -1200,7 +1256,7 @@ mod tests {
expr_fields: &[expr_field],
ignore_nulls: false,
order_bys: &[],
- is_distinct: false,
+ is_distinct,
name: "avg",
is_reversed: false,
exprs: &[],
@@ -1223,6 +1279,15 @@ mod tests {
with_avg_args(input_type, return_type, |args| Avg::new().accumulator(args))
}
+ fn avg_distinct_accumulator(
+ input_type: &DataType,
+ return_type: &DataType,
+ ) -> Result<Box<dyn Accumulator>> {
+ with_avg_args_distinct(input_type, return_type, true, |args| {
+ Avg::new().accumulator(args)
+ })
+ }
+
fn avg_state_fields(
input_type: &DataType,
return_type: &DataType,
@@ -1265,6 +1330,26 @@ mod tests {
sum_type: DataType::Decimal64(18, 0),
expected: ScalarValue::Decimal32(Some(DECIMAL32_VALUE * 10_000), 9, 4),
},
+ AvgCase {
+ name: "decimal32_tpcds_return_amt",
+ values: Arc::new(
+ Decimal32Array::from(vec![100, 200])
+ .with_precision_and_scale(7, 2)?,
+ ),
+ return_type: DataType::Decimal64(11, 6),
+ sum_type: DataType::Decimal128(38, 2),
+ expected: ScalarValue::Decimal64(Some(1_500_000), 11, 6),
+ },
+ AvgCase {
+ name: "decimal64_preserves_output_width",
+ values: Arc::new(
+ Decimal64Array::from(vec![100, 200])
+ .with_precision_and_scale(5, 2)?,
+ ),
+ return_type: DataType::Decimal64(9, 6),
+ sum_type: DataType::Decimal64(18, 2),
+ expected: ScalarValue::Decimal64(Some(1_500_000), 9, 6),
+ },
AvgCase {
name: "decimal64",
values: Arc::new(
@@ -1275,6 +1360,16 @@ mod tests {
sum_type: DataType::Decimal128(38, 0),
expected: ScalarValue::Decimal64(Some(DECIMAL64_VALUE * 10_000), 18, 4),
},
+ AvgCase {
+ name: "decimal64_widened_output",
+ values: Arc::new(
+ Decimal64Array::from(vec![100, 200])
+ .with_precision_and_scale(15, 2)?,
+ ),
+ return_type: DataType::Decimal128(19, 6),
+ sum_type: DataType::Decimal128(38, 2),
+ expected: ScalarValue::Decimal128(Some(1_500_000), 19, 6),
+ },
AvgCase {
name: "decimal128",
values: Arc::new(
@@ -1285,6 +1380,20 @@ mod tests {
sum_type: DataType::Decimal256(76, 0),
expected: ScalarValue::Decimal128(Some(DECIMAL128_VALUE * 10_000), 38, 4),
},
+ AvgCase {
+ name: "decimal128_widened_output",
+ values: Arc::new(
+ Decimal128Array::from(vec![100, 200])
+ .with_precision_and_scale(35, 2)?,
+ ),
+ return_type: DataType::Decimal256(39, 6),
+ sum_type: DataType::Decimal256(76, 2),
+ expected: ScalarValue::Decimal256(
+ Some(i256::from_i128(1_500_000)),
+ 39,
+ 6,
+ ),
+ },
AvgCase {
name: "decimal256",
values: Arc::new(
@@ -1314,9 +1423,9 @@ mod tests {
values: Arc::new(
Decimal32Array::from(vec![10, 20]).with_precision_and_scale(9, 0)?,
),
- return_type: DataType::Decimal32(9, 4),
+ return_type: DataType::Decimal64(13, 4),
sum_type: DataType::Decimal128(38, 0),
- expected: ScalarValue::Decimal32(Some(150_000), 9, 4),
+ expected: ScalarValue::Decimal64(Some(150_000), 13, 4),
},
// One duration unit suffices: all four units instantiate the same
// `S = I` generic code
@@ -1330,6 +1439,79 @@ mod tests {
])
}
+ #[test]
+ fn avg_decimal_return_type_and_dispatch() -> Result<()> {
+ // One case for every input/sum/output combination supported by
+ // `decimal_avg_dispatch`, plus the Decimal256 limit.
+ let cases = [
+ (DataType::Decimal32(5, 2), DataType::Decimal32(9, 6)),
+ (DataType::Decimal32(6, 2), DataType::Decimal64(10, 6)),
+ (DataType::Decimal64(5, 2), DataType::Decimal64(9, 6)),
+ (DataType::Decimal64(6, 2), DataType::Decimal64(10, 6)),
+ (DataType::Decimal64(15, 2), DataType::Decimal128(19, 6)),
+ (DataType::Decimal128(25, 2), DataType::Decimal128(29, 6)),
+ (DataType::Decimal128(26, 2), DataType::Decimal128(30, 6)),
+ (DataType::Decimal128(35, 2), DataType::Decimal256(39, 6)),
+ (DataType::Decimal256(5, 2), DataType::Decimal256(9, 6)),
+ // Precision and scale remain capped at the Decimal256 limits.
+ (DataType::Decimal256(76, 72), DataType::Decimal256(76, 76)),
+ ];
+ let avg = Avg::new();
+
+ for (input, expected) in cases {
+ assert_eq!(avg.return_type(std::slice::from_ref(&input))?, expected);
+
+ let mut acc = avg_accumulator(&input, &expected)?;
+ assert_eq!(acc.evaluate()?.data_type(), expected);
+
+ let mut distinct_acc = avg_distinct_accumulator(&input, &expected)?;
+ assert_eq!(distinct_acc.evaluate()?.data_type(), expected);
+
+ let mut groups_acc = avg_groups_accumulator(&input, &expected)?;
+ assert_eq!(groups_acc.evaluate(EmitTo::All)?.data_type(), &expected);
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn avg_distinct_supports_wider_return_width() -> Result<()> {
+ let cases: [(ArrayRef, DataType, ScalarValue); 3] = [
+ (
+ Arc::new(
+ Decimal32Array::from(vec![100, 200, 200])
+ .with_precision_and_scale(7, 2)?,
+ ),
+ DataType::Decimal64(11, 6),
+ ScalarValue::Decimal64(Some(1_500_000), 11, 6),
+ ),
+ (
+ Arc::new(
+ Decimal64Array::from(vec![100, 200, 200])
+ .with_precision_and_scale(15, 2)?,
+ ),
+ DataType::Decimal128(19, 6),
+ ScalarValue::Decimal128(Some(1_500_000), 19, 6),
+ ),
+ (
+ Arc::new(
+ Decimal128Array::from(vec![100, 200, 200])
+ .with_precision_and_scale(35, 2)?,
+ ),
+ DataType::Decimal256(39, 6),
+ ScalarValue::Decimal256(Some(i256::from_i128(1_500_000)), 39, 6),
+ ),
+ ];
+
+ for (values, return_type, expected) in cases {
+ let mut acc = avg_distinct_accumulator(values.data_type(), &return_type)?;
+ acc.update_batch(&[values])?;
+ assert_eq!(acc.evaluate()?, expected);
+ }
+
+ Ok(())
+ }
+
#[test]
fn avg_accumulator_evaluate_and_state_types() -> Result<()> {
for case in avg_cases()? {
@@ -1402,18 +1584,21 @@ mod tests {
Ok(())
}
- /// The widened sum fits, but the average does not fit the output type once
- /// `DecimalAverager` rescales it: avg must error rather than silently wrap
+ /// The widened return type must hold the rescaled average at the input's
+ /// maximum precision.
#[test]
- fn avg_errors_when_average_exceeds_output_precision() -> Result<()> {
+ fn avg_widens_output_to_fit_rescaled_average() -> Result<()> {
let values: ArrayRef = Arc::new(
Decimal32Array::from(vec![999_999_999]).with_precision_and_scale(9, 0)?,
);
- let return_type = DataType::Decimal32(9, 4);
+ let return_type = Avg::new().return_type(&[values.data_type().clone()])?;
let mut acc = avg_accumulator(values.data_type(), &return_type)?;
acc.update_batch(&[values])?;
- assert!(acc.evaluate().is_err());
+ assert_eq!(
+ acc.evaluate()?,
+ ScalarValue::Decimal64(Some(9_999_999_990_000), 13, 4)
+ );
Ok(())
}
diff --git a/datafusion/sqllogictest/test_files/decimal.slt b/datafusion/sqllogictest/test_files/decimal.slt
index 4335ec0668..c8475272f7 100644
--- a/datafusion/sqllogictest/test_files/decimal.slt
+++ b/datafusion/sqllogictest/test_files/decimal.slt
@@ -1339,4 +1339,33 @@ from (
from generate_series(1, 65536) t(v)
) t;
----
-32768.5 Decimal32(9, 4) 32768.5
+32768.5 Decimal64(13, 4) 32768.5
+
+# TPC-DS `store_returns.sr_return_amt` is Decimal32(7, 2). Its average needs
+# Decimal(11, 6), which requires Decimal64 rather than a clamped Decimal32(9, 6).
+query RT
+select avg(d), arrow_typeof(avg(d))
+from (
+ values
+ (arrow_cast(1.00, 'Decimal32(7, 2)')),
+ (arrow_cast(2.00, 'Decimal32(7, 2)'))
+) t(d);
+----
+1.5 Decimal64(11, 6)
+
+# GROUP BY selects the groups accumulator, which must produce the same widened
+# Decimal64 output type for each group.
+query IRT
+select g, avg(d), arrow_typeof(avg(d))
+from (
+ values
+ (1, arrow_cast(1.00, 'Decimal32(7, 2)')),
+ (1, arrow_cast(2.00, 'Decimal32(7, 2)')),
+ (2, arrow_cast(3.00, 'Decimal32(7, 2)')),
+ (2, arrow_cast(4.00, 'Decimal32(7, 2)'))
+) t(g, d)
+group by g
+order by g;
+----
+1 1.5 Decimal64(11, 6)
+2 3.5 Decimal64(11, 6)
diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt
index e76a3e8147..2a022b947d 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -1908,14 +1908,14 @@ WHERE e1.salary > (
----
logical_plan
01)Projection: e1.employee_name, e1.salary
-02)--LeftSemi Join: e1.dept_id = __scalar_sq_1.dept_id Filter: CAST(e1.salary AS Decimal128(38, 14)) > __scalar_sq_1.avg(e2.salary)
+02)--LeftSemi Join: e1.dept_id = __scalar_sq_1.dept_id Filter: CAST(e1.salary AS Decimal256(42, 14)) > __scalar_sq_1.avg(e2.salary)
03)----SubqueryAlias: e1
04)------TableScan: employees projection=[employee_name, dept_id, salary]
05)----SubqueryAlias: __scalar_sq_1
06)------Projection: avg(e2.salary), e2.dept_id
07)--------Aggregate: groupBy=[[e2.dept_id]], aggr=[[avg(e2.salary)]]
08)----------Projection: e2.dept_id, e2.salary
-09)------------Inner Join: Filter: CAST(e2.salary AS Decimal128(38, 14)) > __scalar_sq_2.avg(e3.salary) AND __scalar_sq_2.dept_id = e1.dept_id
+09)------------Inner Join: Filter: CAST(e2.salary AS Decimal256(42, 14)) > __scalar_sq_2.avg(e3.salary) AND __scalar_sq_2.dept_id = e1.dept_id
10)--------------SubqueryAlias: e2
11)----------------TableScan: employees projection=[dept_id, salary]
12)--------------SubqueryAlias: __scalar_sq_2