Skip to content

Apache DataFusion SQL Query Engine

chore: remove unreachable code and cleanup data type matching + support interval in min/max groups acc (#25872)

Raz Luvatonec10fc9cf570
diff --git a/datafusion/functions-aggregate/src/min_max.rs b/datafusion/functions-aggregate/src/min_max.rs
index 94814a802d..0cf2837f5e 100644
--- a/datafusion/functions-aggregate/src/min_max.rs
+++ b/datafusion/functions-aggregate/src/min_max.rs
@@ -21,13 +21,8 @@
mod min_max_bytes;
mod min_max_struct;
-use arrow::array::ArrayRef;
-use arrow::datatypes::{
- DataType, Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type,
- DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType,
- DurationSecondType, Float16Type, Float32Type, Float64Type, Int8Type, Int16Type,
- Int32Type, Int64Type, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
-};
+use arrow::array::{ArrayRef, ArrowPrimitiveType, downcast_primitive};
+use arrow::datatypes::{DataType, Float16Type, Float32Type, Float64Type};
use datafusion_common::stats::Precision;
use datafusion_common::{ColumnStatistics, Result, exec_err, internal_err};
use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;
@@ -35,13 +30,6 @@ use datafusion_physical_expr::expressions;
use std::cmp::Ordering;
use std::fmt::Debug;
-use arrow::datatypes::i256;
-use arrow::datatypes::{
- Date32Type, Date64Type, Time32MillisecondType, Time32SecondType,
- Time64MicrosecondType, Time64NanosecondType, TimeUnit, TimestampMicrosecondType,
- TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,
-};
-
use crate::min_max::min_max_bytes::MinMaxBytesAccumulator;
use crate::min_max::min_max_struct::MinMaxStructAccumulator;
use datafusion_common::ScalarValue;
@@ -52,7 +40,6 @@ use datafusion_expr::{
};
use datafusion_expr::{GroupsAccumulator, StatisticsArgs};
use datafusion_macros::user_doc;
-use half::f16;
use std::collections::VecDeque;
use std::mem::{size_of, size_of_val};
use std::ops::Deref;
@@ -116,9 +103,9 @@ impl Default for Max {
///
/// [`ArrowPrimitiveType`]: arrow::datatypes::ArrowPrimitiveType
macro_rules! primitive_max_accumulator {
- ($DATA_TYPE:ident, $NATIVE:ident, $PRIMTYPE:ident) => {{
+ ($t:ty, $data_type:ident) => {{
Ok(Box::new(
- PrimitiveGroupsAccumulator::<$PRIMTYPE, _>::new($DATA_TYPE, |cur, new| {
+ PrimitiveGroupsAccumulator::<$t, _>::new($data_type, |cur, new| {
match (new).partial_cmp(cur) {
Some(Ordering::Greater) | None => {
// new is Greater or None
@@ -127,19 +114,21 @@ macro_rules! primitive_max_accumulator {
_ => {}
}
})
- // Initialize each accumulator to $NATIVE::MIN
- .with_starting_value($NATIVE::MIN),
+ // Initialize each accumulator to $t::Native::MIN
+ .with_starting_value(<$t as ArrowPrimitiveType>::Native::MIN),
))
}};
- ($DATA_TYPE:ident, $NATIVE:ident, $PRIMTYPE:ident, total, $BITS:ident) => {{
+ ($t:ty, $data_type:ident, total, $bits:ident) => {{
Ok(Box::new(
- PrimitiveGroupsAccumulator::<$PRIMTYPE, _>::new($DATA_TYPE, |cur, new| {
+ PrimitiveGroupsAccumulator::<$t, _>::new($data_type, |cur, new| {
if new.total_cmp(cur) == Ordering::Greater {
*cur = new
}
})
// Use the total-order minimum so negative NaNs replace the sentinel.
- .with_starting_value($NATIVE::from_bits($BITS::MAX)),
+ .with_starting_value(
+ <$t as ArrowPrimitiveType>::Native::from_bits($bits::MAX),
+ ),
))
}};
}
@@ -150,9 +139,9 @@ macro_rules! primitive_max_accumulator {
///
/// [`ArrowPrimitiveType`]: arrow::datatypes::ArrowPrimitiveType
macro_rules! primitive_min_accumulator {
- ($DATA_TYPE:ident, $NATIVE:ident, $PRIMTYPE:ident) => {{
+ ($t:ty, $data_type:ident) => {{
Ok(Box::new(
- PrimitiveGroupsAccumulator::<$PRIMTYPE, _>::new(&$DATA_TYPE, |cur, new| {
+ PrimitiveGroupsAccumulator::<$t, _>::new(&$data_type, |cur, new| {
match (new).partial_cmp(cur) {
Some(Ordering::Less) | None => {
// new is Less or NaN
@@ -162,18 +151,20 @@ macro_rules! primitive_min_accumulator {
}
})
// Initialize each accumulator to $NATIVE::MAX
- .with_starting_value($NATIVE::MAX),
+ .with_starting_value(<$t as ArrowPrimitiveType>::Native::MAX),
))
}};
- ($DATA_TYPE:ident, $NATIVE:ident, $PRIMTYPE:ident, total, $BITS:ident) => {{
+ ($t:ty, $data_type:ident, total, $bits:ident) => {{
Ok(Box::new(
- PrimitiveGroupsAccumulator::<$PRIMTYPE, _>::new(&$DATA_TYPE, |cur, new| {
+ PrimitiveGroupsAccumulator::<$t, _>::new(&$data_type, |cur, new| {
if new.total_cmp(cur) == Ordering::Less {
*cur = new
}
})
// Use the total-order maximum so positive NaNs replace the sentinel.
- .with_starting_value($NATIVE::from_bits($BITS::MAX >> 1)),
+ .with_starting_value(
+ <$t as ArrowPrimitiveType>::Native::from_bits($bits::MAX >> 1),
+ ),
))
}};
}
@@ -246,36 +237,18 @@ impl AggregateUDFImpl for Max {
fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {
use DataType::*;
- matches!(
- args.return_field.data_type(),
- Int8 | Int16
- | Int32
- | Int64
- | UInt8
- | UInt16
- | UInt32
- | UInt64
- | Float16
- | Float32
- | Float64
- | Decimal32(_, _)
- | Decimal64(_, _)
- | Decimal128(_, _)
- | Decimal256(_, _)
- | Date32
- | Date64
- | Time32(_)
- | Time64(_)
- | Timestamp(_, _)
- | Utf8
- | LargeUtf8
- | Utf8View
- | Binary
- | LargeBinary
- | BinaryView
- | Duration(_)
- | Struct(_)
- )
+ let data_type = args.return_field.data_type();
+ data_type.is_primitive()
+ || matches!(
+ data_type,
+ Utf8 | LargeUtf8
+ | Utf8View
+ | Binary
+ | LargeBinary
+ | BinaryView
+ | Duration(_)
+ | Struct(_)
+ )
}
fn create_groups_accumulator(
@@ -283,84 +256,31 @@ impl AggregateUDFImpl for Max {
args: AccumulatorArgs,
) -> Result<Box<dyn GroupsAccumulator>> {
use DataType::*;
- use TimeUnit::*;
let data_type = args.return_field.data_type();
+
match data_type {
- Int8 => primitive_max_accumulator!(data_type, i8, Int8Type),
- Int16 => primitive_max_accumulator!(data_type, i16, Int16Type),
- Int32 => primitive_max_accumulator!(data_type, i32, Int32Type),
- Int64 => primitive_max_accumulator!(data_type, i64, Int64Type),
- UInt8 => primitive_max_accumulator!(data_type, u8, UInt8Type),
- UInt16 => primitive_max_accumulator!(data_type, u16, UInt16Type),
- UInt32 => primitive_max_accumulator!(data_type, u32, UInt32Type),
- UInt64 => primitive_max_accumulator!(data_type, u64, UInt64Type),
Float16 => {
- primitive_max_accumulator!(data_type, f16, Float16Type, total, u16)
+ primitive_max_accumulator!(Float16Type, data_type, total, u16)
}
Float32 => {
- primitive_max_accumulator!(data_type, f32, Float32Type, total, u32)
+ primitive_max_accumulator!(Float32Type, data_type, total, u32)
}
Float64 => {
- primitive_max_accumulator!(data_type, f64, Float64Type, total, u64)
- }
- Date32 => primitive_max_accumulator!(data_type, i32, Date32Type),
- Date64 => primitive_max_accumulator!(data_type, i64, Date64Type),
- Time32(Second) => {
- primitive_max_accumulator!(data_type, i32, Time32SecondType)
- }
- Time32(Millisecond) => {
- primitive_max_accumulator!(data_type, i32, Time32MillisecondType)
- }
- Time64(Microsecond) => {
- primitive_max_accumulator!(data_type, i64, Time64MicrosecondType)
- }
- Time64(Nanosecond) => {
- primitive_max_accumulator!(data_type, i64, Time64NanosecondType)
- }
- Timestamp(Second, _) => {
- primitive_max_accumulator!(data_type, i64, TimestampSecondType)
- }
- Timestamp(Millisecond, _) => {
- primitive_max_accumulator!(data_type, i64, TimestampMillisecondType)
- }
- Timestamp(Microsecond, _) => {
- primitive_max_accumulator!(data_type, i64, TimestampMicrosecondType)
- }
- Timestamp(Nanosecond, _) => {
- primitive_max_accumulator!(data_type, i64, TimestampNanosecondType)
- }
- Duration(Second) => {
- primitive_max_accumulator!(data_type, i64, DurationSecondType)
- }
- Duration(Millisecond) => {
- primitive_max_accumulator!(data_type, i64, DurationMillisecondType)
+ primitive_max_accumulator!(Float64Type, data_type, total, u64)
}
- Duration(Microsecond) => {
- primitive_max_accumulator!(data_type, i64, DurationMicrosecondType)
- }
- Duration(Nanosecond) => {
- primitive_max_accumulator!(data_type, i64, DurationNanosecondType)
- }
- Decimal32(_, _) => {
- primitive_max_accumulator!(data_type, i32, Decimal32Type)
- }
- Decimal64(_, _) => {
- primitive_max_accumulator!(data_type, i64, Decimal64Type)
- }
- Decimal128(_, _) => {
- primitive_max_accumulator!(data_type, i128, Decimal128Type)
- }
- Decimal256(_, _) => {
- primitive_max_accumulator!(data_type, i256, Decimal256Type)
- }
- Utf8 | LargeUtf8 | Utf8View | Binary | LargeBinary | BinaryView => {
- Ok(Box::new(MinMaxBytesAccumulator::new_max(data_type.clone())))
+ _ => {
+ downcast_primitive! {
+ data_type => (primitive_max_accumulator, data_type),
+ Utf8 | LargeUtf8 | Utf8View | Binary | LargeBinary | BinaryView => {
+ Ok(Box::new(MinMaxBytesAccumulator::new_max(data_type.clone())))
+ }
+ Struct(_) => Ok(Box::new(MinMaxStructAccumulator::new_max(
+ data_type.clone(),
+ ))),
+ // This is only reached if groups_accumulator_supported is out of sync
+ _ => internal_err!("GroupsAccumulator not supported for max({})", data_type),
+ }
}
- Struct(_) => Ok(Box::new(MinMaxStructAccumulator::new_max(
- data_type.clone(),
- ))),
- // This is only reached if groups_accumulator_supported is out of sync
- _ => internal_err!("GroupsAccumulator not supported for max({})", data_type),
}
}
@@ -545,36 +465,19 @@ impl AggregateUDFImpl for Min {
fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {
use DataType::*;
- matches!(
- args.return_field.data_type(),
- Int8 | Int16
- | Int32
- | Int64
- | UInt8
- | UInt16
- | UInt32
- | UInt64
- | Float16
- | Float32
- | Float64
- | Decimal32(_, _)
- | Decimal64(_, _)
- | Decimal128(_, _)
- | Decimal256(_, _)
- | Date32
- | Date64
- | Time32(_)
- | Time64(_)
- | Timestamp(_, _)
- | Utf8
- | LargeUtf8
- | Utf8View
- | Binary
- | LargeBinary
- | BinaryView
- | Duration(_)
- | Struct(_)
- )
+
+ let data_type = args.return_field.data_type();
+ data_type.is_primitive()
+ || matches!(
+ data_type,
+ Utf8 | LargeUtf8
+ | Utf8View
+ | Binary
+ | LargeBinary
+ | BinaryView
+ | Duration(_)
+ | Struct(_)
+ )
}
fn create_groups_accumulator(
@@ -582,84 +485,31 @@ impl AggregateUDFImpl for Min {
args: AccumulatorArgs,
) -> Result<Box<dyn GroupsAccumulator>> {
use DataType::*;
- use TimeUnit::*;
let data_type = args.return_field.data_type();
+
match data_type {
- Int8 => primitive_min_accumulator!(data_type, i8, Int8Type),
- Int16 => primitive_min_accumulator!(data_type, i16, Int16Type),
- Int32 => primitive_min_accumulator!(data_type, i32, Int32Type),
- Int64 => primitive_min_accumulator!(data_type, i64, Int64Type),
- UInt8 => primitive_min_accumulator!(data_type, u8, UInt8Type),
- UInt16 => primitive_min_accumulator!(data_type, u16, UInt16Type),
- UInt32 => primitive_min_accumulator!(data_type, u32, UInt32Type),
- UInt64 => primitive_min_accumulator!(data_type, u64, UInt64Type),
Float16 => {
- primitive_min_accumulator!(data_type, f16, Float16Type, total, u16)
+ primitive_min_accumulator!(Float16Type, data_type, total, u16)
}
Float32 => {
- primitive_min_accumulator!(data_type, f32, Float32Type, total, u32)
+ primitive_min_accumulator!(Float32Type, data_type, total, u32)
}
Float64 => {
- primitive_min_accumulator!(data_type, f64, Float64Type, total, u64)
- }
- Date32 => primitive_min_accumulator!(data_type, i32, Date32Type),
- Date64 => primitive_min_accumulator!(data_type, i64, Date64Type),
- Time32(Second) => {
- primitive_min_accumulator!(data_type, i32, Time32SecondType)
- }
- Time32(Millisecond) => {
- primitive_min_accumulator!(data_type, i32, Time32MillisecondType)
- }
- Time64(Microsecond) => {
- primitive_min_accumulator!(data_type, i64, Time64MicrosecondType)
+ primitive_min_accumulator!(Float64Type, data_type, total, u64)
}
- Time64(Nanosecond) => {
- primitive_min_accumulator!(data_type, i64, Time64NanosecondType)
- }
- Timestamp(Second, _) => {
- primitive_min_accumulator!(data_type, i64, TimestampSecondType)
- }
- Timestamp(Millisecond, _) => {
- primitive_min_accumulator!(data_type, i64, TimestampMillisecondType)
- }
- Timestamp(Microsecond, _) => {
- primitive_min_accumulator!(data_type, i64, TimestampMicrosecondType)
- }
- Timestamp(Nanosecond, _) => {
- primitive_min_accumulator!(data_type, i64, TimestampNanosecondType)
- }
- Duration(Second) => {
- primitive_min_accumulator!(data_type, i64, DurationSecondType)
- }
- Duration(Millisecond) => {
- primitive_min_accumulator!(data_type, i64, DurationMillisecondType)
- }
- Duration(Microsecond) => {
- primitive_min_accumulator!(data_type, i64, DurationMicrosecondType)
- }
- Duration(Nanosecond) => {
- primitive_min_accumulator!(data_type, i64, DurationNanosecondType)
- }
- Decimal32(_, _) => {
- primitive_min_accumulator!(data_type, i32, Decimal32Type)
- }
- Decimal64(_, _) => {
- primitive_min_accumulator!(data_type, i64, Decimal64Type)
- }
- Decimal128(_, _) => {
- primitive_min_accumulator!(data_type, i128, Decimal128Type)
- }
- Decimal256(_, _) => {
- primitive_min_accumulator!(data_type, i256, Decimal256Type)
- }
- Utf8 | LargeUtf8 | Utf8View | Binary | LargeBinary | BinaryView => {
- Ok(Box::new(MinMaxBytesAccumulator::new_min(data_type.clone())))
+ _ => {
+ downcast_primitive! {
+ data_type => (primitive_min_accumulator, data_type),
+ Utf8 | LargeUtf8 | Utf8View | Binary | LargeBinary | BinaryView => {
+ Ok(Box::new(MinMaxBytesAccumulator::new_min(data_type.clone())))
+ }
+ Struct(_) => Ok(Box::new(MinMaxStructAccumulator::new_min(
+ data_type.clone(),
+ ))),
+ // This is only reached if groups_accumulator_supported is out of sync
+ _ => internal_err!("GroupsAccumulator not supported for min({})", data_type),
+ }
}
- Struct(_) => Ok(Box::new(MinMaxStructAccumulator::new_min(
- data_type.clone(),
- ))),
- // This is only reached if groups_accumulator_supported is out of sync
- _ => internal_err!("GroupsAccumulator not supported for min({})", data_type),
}
}
@@ -1052,8 +902,8 @@ mod tests {
PrimitiveArray, StringArray,
},
datatypes::{
- ArrowDictionaryKeyType, IntervalDayTimeType, IntervalMonthDayNanoType,
- IntervalUnit, IntervalYearMonthType,
+ ArrowDictionaryKeyType, Field, Float32Type, IntervalDayTimeType,
+ IntervalMonthDayNanoType, IntervalUnit, IntervalYearMonthType, Schema,
},
};
use datafusion_expr::EmitTo;
@@ -1063,12 +913,12 @@ mod tests {
fn grouped_float_min_max_total_order() -> Result<()> {
fn grouped_float32_min() -> Result<Box<dyn GroupsAccumulator>> {
let data_type = &DataType::Float32;
- primitive_min_accumulator!(data_type, f32, Float32Type, total, u32)
+ primitive_min_accumulator!(Float32Type, data_type, total, u32)
}
fn grouped_float32_max() -> Result<Box<dyn GroupsAccumulator>> {
let data_type = &DataType::Float32;
- primitive_max_accumulator!(data_type, f32, Float32Type, total, u32)
+ primitive_max_accumulator!(Float32Type, data_type, total, u32)
}
fn evaluate_grouped_float32(
@@ -1214,6 +1064,180 @@ mod tests {
);
}
+ /// Runs `udf` through its [`GroupsAccumulator`], feeding `values` to two
+ /// accumulators (first and second half of the rows) and merging the state
+ /// of the first into the second, so that `update_batch`, `state` and
+ /// `merge_batch` are all exercised.
+ fn evaluate_grouped(
+ udf: &dyn AggregateUDFImpl,
+ values: &ArrayRef,
+ group_indices: &[usize],
+ total_num_groups: usize,
+ ) -> Result<ArrayRef> {
+ let data_type = values.data_type().clone();
+ let schema = Schema::new(vec![Field::new("v", data_type.clone(), true)]);
+ let expr: Arc<dyn datafusion_physical_expr::PhysicalExpr> =
+ Arc::new(expressions::Column::new("v", 0));
+ let expr_field = Arc::new(Field::new("v", data_type.clone(), true));
+ let args = || AccumulatorArgs {
+ schema: &schema,
+ expr_fields: std::slice::from_ref(&expr_field),
+ exprs: std::slice::from_ref(&expr),
+ is_distinct: false,
+ name: udf.name(),
+ ignore_nulls: false,
+ is_reversed: false,
+ return_field: Arc::new(Field::new(udf.name(), data_type.clone(), true)),
+ order_bys: &[],
+ };
+
+ let mid = values.len() / 2;
+ let mut first = udf.create_groups_accumulator(args())?;
+ first.update_batch(
+ &[values.slice(0, mid)],
+ &group_indices[..mid],
+ None,
+ total_num_groups,
+ )?;
+
+ let mut second = udf.create_groups_accumulator(args())?;
+ second.update_batch(
+ &[values.slice(mid, values.len() - mid)],
+ &group_indices[mid..],
+ None,
+ total_num_groups,
+ )?;
+
+ let state = first.state(EmitTo::All)?;
+ let merge_group_indices: Vec<usize> = (0..total_num_groups).collect();
+ second.merge_batch(&state, &merge_group_indices, total_num_groups)?;
+
+ let result = second.evaluate(EmitTo::All)?;
+ assert_eq!(result.data_type(), &data_type);
+ Ok(result)
+ }
+
+ #[test]
+ fn interval_min_max_groups_accumulator() -> Result<()> {
+ // Group 0 and 1 get values from both halves of the input, group 2 only
+ // sees nulls and group 3 never receives any row.
+ let group_indices = [0, 1, 2, 0, 1, 0, 1, 2];
+ let total_num_groups = 4;
+
+ let cases: Vec<(ArrayRef, ArrayRef, ArrayRef)> = vec![
+ (
+ Arc::new(IntervalYearMonthArray::from(vec![
+ Some(IntervalYearMonthType::make_value(0, 1)),
+ Some(IntervalYearMonthType::make_value(5, 34)),
+ None,
+ Some(IntervalYearMonthType::make_value(-2, 4)),
+ Some(IntervalYearMonthType::make_value(7, -4)),
+ Some(IntervalYearMonthType::make_value(0, 1)),
+ None,
+ None,
+ ])),
+ // min
+ Arc::new(IntervalYearMonthArray::from(vec![
+ Some(IntervalYearMonthType::make_value(-2, 4)),
+ Some(IntervalYearMonthType::make_value(7, -4)),
+ None,
+ None,
+ ])),
+ // max
+ Arc::new(IntervalYearMonthArray::from(vec![
+ Some(IntervalYearMonthType::make_value(0, 1)),
+ Some(IntervalYearMonthType::make_value(5, 34)),
+ None,
+ None,
+ ])),
+ ),
+ (
+ Arc::new(IntervalDayTimeArray::from(vec![
+ Some(IntervalDayTimeType::make_value(0, 0)),
+ Some(IntervalDayTimeType::make_value(5, 454000)),
+ None,
+ Some(IntervalDayTimeType::make_value(-34, 0)),
+ Some(IntervalDayTimeType::make_value(7, -4000)),
+ Some(IntervalDayTimeType::make_value(1, 0)),
+ Some(IntervalDayTimeType::make_value(5, 453999)),
+ None,
+ ])),
+ // min
+ Arc::new(IntervalDayTimeArray::from(vec![
+ Some(IntervalDayTimeType::make_value(-34, 0)),
+ Some(IntervalDayTimeType::make_value(5, 453999)),
+ None,
+ None,
+ ])),
+ // max
+ Arc::new(IntervalDayTimeArray::from(vec![
+ Some(IntervalDayTimeType::make_value(1, 0)),
+ Some(IntervalDayTimeType::make_value(7, -4000)),
+ None,
+ None,
+ ])),
+ ),
+ (
+ Arc::new(IntervalMonthDayNanoArray::from(vec![
+ Some(IntervalMonthDayNanoType::make_value(1, 0, 0)),
+ Some(IntervalMonthDayNanoType::make_value(
+ 344,
+ 34,
+ -43_000_000_000,
+ )),
+ None,
+ Some(IntervalMonthDayNanoType::make_value(
+ -593,
+ -33,
+ 13_000_000_000,
+ )),
+ Some(IntervalMonthDayNanoType::make_value(5, 2, 493_000_000_000)),
+ Some(IntervalMonthDayNanoType::make_value(1, 0, 1)),
+ Some(IntervalMonthDayNanoType::make_value(
+ 344,
+ 34,
+ -42_000_000_000,
+ )),
+ None,
+ ])),
+ // min
+ Arc::new(IntervalMonthDayNanoArray::from(vec![
+ Some(IntervalMonthDayNanoType::make_value(
+ -593,
+ -33,
+ 13_000_000_000,
+ )),
+ Some(IntervalMonthDayNanoType::make_value(5, 2, 493_000_000_000)),
+ None,
+ None,
+ ])),
+ // max
+ Arc::new(IntervalMonthDayNanoArray::from(vec![
+ Some(IntervalMonthDayNanoType::make_value(1, 0, 1)),
+ Some(IntervalMonthDayNanoType::make_value(
+ 344,
+ 34,
+ -42_000_000_000,
+ )),
+ None,
+ None,
+ ])),
+ ),
+ ];
+
+ for (values, expected_min, expected_max) in cases {
+ let min =
+ evaluate_grouped(&Min::new(), &values, &group_indices, total_num_groups)?;
+ assert_eq!(&min, &expected_min, "min({})", values.data_type());
+
+ let max =
+ evaluate_grouped(&Max::new(), &values, &group_indices, total_num_groups)?;
+ assert_eq!(&max, &expected_max, "max({})", values.data_type());
+ }
+
+ Ok(())
+ }
+
#[test]
fn float_min_max_with_nans() {
let pos_nan = f32::NAN;
diff --git a/datafusion/physical-plan/src/aggregates/group_values/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/mod.rs
index 80e6b335e3..e03db633e6 100644
--- a/datafusion/physical-plan/src/aggregates/group_values/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/group_values/mod.rs
@@ -17,13 +17,8 @@
//! [`GroupValues`] trait for storing and interning group keys
-use arrow::array::types::{
- Date32Type, Date64Type, Decimal128Type, Time32MillisecondType, Time32SecondType,
- Time64MicrosecondType, Time64NanosecondType, TimestampMicrosecondType,
- TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,
-};
use arrow::array::{ArrayRef, downcast_primitive};
-use arrow::datatypes::{DataType, SchemaRef, TimeUnit};
+use arrow::datatypes::{DataType, SchemaRef};
use datafusion_common::{Result, not_impl_err};
use datafusion_expr::{EmitTo, GroupSelection};
@@ -171,35 +166,6 @@ pub fn new_group_values(
downcast_primitive! {
d => (downcast_helper, d),
- _ => {}
- }
-
- match d {
- DataType::Date32 => {
- downcast_helper!(Date32Type, d);
- }
- DataType::Date64 => {
- downcast_helper!(Date64Type, d);
- }
- DataType::Time32(t) => match t {
- TimeUnit::Second => downcast_helper!(Time32SecondType, d),
- TimeUnit::Millisecond => downcast_helper!(Time32MillisecondType, d),
- _ => {}
- },
- DataType::Time64(t) => match t {
- TimeUnit::Microsecond => downcast_helper!(Time64MicrosecondType, d),
- TimeUnit::Nanosecond => downcast_helper!(Time64NanosecondType, d),
- _ => {}
- },
- DataType::Timestamp(t, _tz) => match t {
- TimeUnit::Second => downcast_helper!(TimestampSecondType, d),
- TimeUnit::Millisecond => downcast_helper!(TimestampMillisecondType, d),
- TimeUnit::Microsecond => downcast_helper!(TimestampMicrosecondType, d),
- TimeUnit::Nanosecond => downcast_helper!(TimestampNanosecondType, d),
- },
- DataType::Decimal128(_, _) => {
- downcast_helper!(Decimal128Type, d);
- }
DataType::Utf8 => {
return Ok(Box::new(GroupValuesBytes::<i32>::new(OutputType::Utf8)));
}
diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs
index 098b49dd4f..07b825e95a 100644
--- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs
@@ -36,17 +36,12 @@ use crate::aggregates::group_values::multi_group_by::{
fixed_size_binary::FixedSizeBinaryGroupValueBuilder,
primitive::PrimitiveGroupValueBuilder, row_backed::RowsGroupColumn,
};
-use arrow::array::{Array, ArrayRef, BooleanBufferBuilder, new_empty_array};
+use arrow::array::{
+ Array, ArrayRef, BooleanBufferBuilder, downcast_primitive, new_empty_array,
+};
use arrow::datatypes::{
- BinaryViewType, DataType, Date32Type, Date64Type, Decimal32Type, Decimal64Type,
- Decimal128Type, Decimal256Type, DurationMicrosecondType, DurationMillisecondType,
- DurationNanosecondType, DurationSecondType, Field, Float16Type, Float32Type,
- Float64Type, Int8Type, Int16Type, Int32Type, Int64Type, IntervalDayTimeType,
- IntervalMonthDayNanoType, IntervalUnit, IntervalYearMonthType, Schema, SchemaRef,
- StringViewType, Time32MillisecondType, Time32SecondType, Time64MicrosecondType,
- Time64NanosecondType, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,
- TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type,
- UInt64Type,
+ BinaryViewType, DataType, Field, Int8Type, Int16Type, Int32Type, Int64Type, Schema,
+ SchemaRef, StringViewType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
};
use datafusion_common::hash_utils::RandomState;
use datafusion_common::hash_utils::create_hashes;
@@ -913,11 +908,11 @@ impl<const STREAMING: bool> GroupValuesColumn<STREAMING> {
/// instantiates a [`PrimitiveGroupValueBuilder`] and pushes it into $v
///
/// Arguments:
+/// `$t`: the primitive type of the builder
/// `$v`: the vector to push the new builder into
/// `$nullable`: whether the input can contains nulls
-/// `$t`: the primitive type of the builder
macro_rules! instantiate_primitive {
- ($nullable:expr, $t:ty, $data_type:ident) => {{
+ ($t:ty, $data_type:ident, $nullable:expr) => {{
let builder: Box<dyn GroupColumn> = if $nullable {
Box::new(PrimitiveGroupValueBuilder::<$t, true>::new(
$data_type.to_owned(),
@@ -959,95 +954,10 @@ fn group_column_supported_type(data_type: &DataType) -> bool {
fn make_group_column(field: &Field) -> Result<Box<dyn GroupColumn>> {
let nullable = field.is_nullable();
let data_type = field.data_type();
- let builder: Option<Box<dyn GroupColumn>> = match data_type {
- DataType::Int8 => instantiate_primitive!(nullable, Int8Type, data_type),
- DataType::Int16 => instantiate_primitive!(nullable, Int16Type, data_type),
- DataType::Int32 => instantiate_primitive!(nullable, Int32Type, data_type),
- DataType::Int64 => instantiate_primitive!(nullable, Int64Type, data_type),
- DataType::UInt8 => instantiate_primitive!(nullable, UInt8Type, data_type),
- DataType::UInt16 => instantiate_primitive!(nullable, UInt16Type, data_type),
- DataType::UInt32 => instantiate_primitive!(nullable, UInt32Type, data_type),
- DataType::UInt64 => instantiate_primitive!(nullable, UInt64Type, data_type),
- DataType::Float16 => instantiate_primitive!(nullable, Float16Type, data_type),
- DataType::Float32 => instantiate_primitive!(nullable, Float32Type, data_type),
- DataType::Float64 => instantiate_primitive!(nullable, Float64Type, data_type),
- DataType::Date32 => instantiate_primitive!(nullable, Date32Type, data_type),
- DataType::Date64 => instantiate_primitive!(nullable, Date64Type, data_type),
- DataType::Time32(t) => match t {
- TimeUnit::Second => {
- instantiate_primitive!(nullable, Time32SecondType, data_type)
- }
- TimeUnit::Millisecond => {
- instantiate_primitive!(nullable, Time32MillisecondType, data_type)
- }
- // Time32 with Microsecond / Nanosecond is not a valid Arrow type
- // combination; reject explicitly.
- _ => None,
- },
- DataType::Time64(t) => match t {
- TimeUnit::Microsecond => {
- instantiate_primitive!(nullable, Time64MicrosecondType, data_type)
- }
- TimeUnit::Nanosecond => {
- instantiate_primitive!(nullable, Time64NanosecondType, data_type)
- }
- // Time64 with Second / Millisecond is not a valid Arrow type
- // combination; reject explicitly.
- _ => None,
- },
- DataType::Timestamp(t, _) => match t {
- TimeUnit::Second => {
- instantiate_primitive!(nullable, TimestampSecondType, data_type)
- }
- TimeUnit::Millisecond => {
- instantiate_primitive!(nullable, TimestampMillisecondType, data_type)
- }
- TimeUnit::Microsecond => {
- instantiate_primitive!(nullable, TimestampMicrosecondType, data_type)
- }
- TimeUnit::Nanosecond => {
- instantiate_primitive!(nullable, TimestampNanosecondType, data_type)
- }
- },
- DataType::Duration(t) => match t {
- TimeUnit::Second => {
- instantiate_primitive!(nullable, DurationSecondType, data_type)
- }
- TimeUnit::Millisecond => {
- instantiate_primitive!(nullable, DurationMillisecondType, data_type)
- }
- TimeUnit::Microsecond => {
- instantiate_primitive!(nullable, DurationMicrosecondType, data_type)
- }
- TimeUnit::Nanosecond => {
- instantiate_primitive!(nullable, DurationNanosecondType, data_type)
- }
- },
- // `IntervalUnit` has exactly three variants, so this match is exhaustive
- // with no fallback arm (unlike Time32 / Time64).
- DataType::Interval(u) => match u {
- IntervalUnit::YearMonth => {
- instantiate_primitive!(nullable, IntervalYearMonthType, data_type)
- }
- IntervalUnit::DayTime => {
- instantiate_primitive!(nullable, IntervalDayTimeType, data_type)
- }
- IntervalUnit::MonthDayNano => {
- instantiate_primitive!(nullable, IntervalMonthDayNanoType, data_type)
- }
- },
- DataType::Decimal32(_, _) => {
- instantiate_primitive!(nullable, Decimal32Type, data_type)
- }
- DataType::Decimal64(_, _) => {
- instantiate_primitive!(nullable, Decimal64Type, data_type)
- }
- DataType::Decimal128(_, _) => {
- instantiate_primitive!(nullable, Decimal128Type, data_type)
- }
- DataType::Decimal256(_, _) => {
- instantiate_primitive!(nullable, Decimal256Type, data_type)
- }
+
+ let builder: Option<Box<dyn GroupColumn>> = downcast_primitive! {
+ data_type => (instantiate_primitive, data_type, nullable),
+
DataType::Utf8 => Some(Box::new(ByteGroupValueBuilder::<i32>::new(
OutputType::Utf8,
))),