Apache DataFusion SQL Query Engine
chore: remove unreachable code and cleanup data type matching + support interval in min/max groups acc (#25872)
Raz Luvatonec10fc9cf570
diff --git a/datafusion/functions-aggregate/src/min_max.rs b/datafusion/functions-aggregate/src/min_max.rsindex 94814a802d..0cf2837f5e 100644--- a/datafusion/functions-aggregate/src/min_max.rs+++ b/datafusion/functions-aggregate/src/min_max.rs@@ -21,13 +21,8 @@mod min_max_bytes;mod min_max_struct;-use arrow::array::ArrayRef;-use arrow::datatypes::{- DataType, Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type,- DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType,- DurationSecondType, Float16Type, Float32Type, Float64Type, Int8Type, Int16Type,- Int32Type, Int64Type, UInt8Type, UInt16Type, UInt32Type, UInt64Type,-};+use arrow::array::{ArrayRef, ArrowPrimitiveType, downcast_primitive};+use arrow::datatypes::{DataType, Float16Type, Float32Type, Float64Type};use datafusion_common::stats::Precision;use datafusion_common::{ColumnStatistics, Result, exec_err, internal_err};use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;@@ -35,13 +30,6 @@ use datafusion_physical_expr::expressions;use std::cmp::Ordering;use std::fmt::Debug;-use arrow::datatypes::i256;-use arrow::datatypes::{- Date32Type, Date64Type, Time32MillisecondType, Time32SecondType,- Time64MicrosecondType, Time64NanosecondType, TimeUnit, TimestampMicrosecondType,- TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,-};-use crate::min_max::min_max_bytes::MinMaxBytesAccumulator;use crate::min_max::min_max_struct::MinMaxStructAccumulator;use datafusion_common::ScalarValue;@@ -52,7 +40,6 @@ use datafusion_expr::{};use datafusion_expr::{GroupsAccumulator, StatisticsArgs};use datafusion_macros::user_doc;-use half::f16;use std::collections::VecDeque;use std::mem::{size_of, size_of_val};use std::ops::Deref;@@ -116,9 +103,9 @@ impl Default for Max {////// [`ArrowPrimitiveType`]: arrow::datatypes::ArrowPrimitiveTypemacro_rules! primitive_max_accumulator {- ($DATA_TYPE:ident, $NATIVE:ident, $PRIMTYPE:ident) => {{+ ($t:ty, $data_type:ident) => {{Ok(Box::new(- PrimitiveGroupsAccumulator::<$PRIMTYPE, _>::new($DATA_TYPE, |cur, new| {+ PrimitiveGroupsAccumulator::<$t, _>::new($data_type, |cur, new| {match (new).partial_cmp(cur) {Some(Ordering::Greater) | None => {// new is Greater or None@@ -127,19 +114,21 @@ macro_rules! primitive_max_accumulator {_ => {}}})- // Initialize each accumulator to $NATIVE::MIN- .with_starting_value($NATIVE::MIN),+ // Initialize each accumulator to $t::Native::MIN+ .with_starting_value(<$t as ArrowPrimitiveType>::Native::MIN),))}};- ($DATA_TYPE:ident, $NATIVE:ident, $PRIMTYPE:ident, total, $BITS:ident) => {{+ ($t:ty, $data_type:ident, total, $bits:ident) => {{Ok(Box::new(- PrimitiveGroupsAccumulator::<$PRIMTYPE, _>::new($DATA_TYPE, |cur, new| {+ PrimitiveGroupsAccumulator::<$t, _>::new($data_type, |cur, new| {if new.total_cmp(cur) == Ordering::Greater {*cur = new}})// Use the total-order minimum so negative NaNs replace the sentinel.- .with_starting_value($NATIVE::from_bits($BITS::MAX)),+ .with_starting_value(+ <$t as ArrowPrimitiveType>::Native::from_bits($bits::MAX),+ ),))}};}@@ -150,9 +139,9 @@ macro_rules! primitive_max_accumulator {////// [`ArrowPrimitiveType`]: arrow::datatypes::ArrowPrimitiveTypemacro_rules! primitive_min_accumulator {- ($DATA_TYPE:ident, $NATIVE:ident, $PRIMTYPE:ident) => {{+ ($t:ty, $data_type:ident) => {{Ok(Box::new(- PrimitiveGroupsAccumulator::<$PRIMTYPE, _>::new(&$DATA_TYPE, |cur, new| {+ PrimitiveGroupsAccumulator::<$t, _>::new(&$data_type, |cur, new| {match (new).partial_cmp(cur) {Some(Ordering::Less) | None => {// new is Less or NaN@@ -162,18 +151,20 @@ macro_rules! primitive_min_accumulator {}})// Initialize each accumulator to $NATIVE::MAX- .with_starting_value($NATIVE::MAX),+ .with_starting_value(<$t as ArrowPrimitiveType>::Native::MAX),))}};- ($DATA_TYPE:ident, $NATIVE:ident, $PRIMTYPE:ident, total, $BITS:ident) => {{+ ($t:ty, $data_type:ident, total, $bits:ident) => {{Ok(Box::new(- PrimitiveGroupsAccumulator::<$PRIMTYPE, _>::new(&$DATA_TYPE, |cur, new| {+ PrimitiveGroupsAccumulator::<$t, _>::new(&$data_type, |cur, new| {if new.total_cmp(cur) == Ordering::Less {*cur = new}})// Use the total-order maximum so positive NaNs replace the sentinel.- .with_starting_value($NATIVE::from_bits($BITS::MAX >> 1)),+ .with_starting_value(+ <$t as ArrowPrimitiveType>::Native::from_bits($bits::MAX >> 1),+ ),))}};}@@ -246,36 +237,18 @@ impl AggregateUDFImpl for Max {fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {use DataType::*;- matches!(- args.return_field.data_type(),- Int8 | Int16- | Int32- | Int64- | UInt8- | UInt16- | UInt32- | UInt64- | Float16- | Float32- | Float64- | Decimal32(_, _)- | Decimal64(_, _)- | Decimal128(_, _)- | Decimal256(_, _)- | Date32- | Date64- | Time32(_)- | Time64(_)- | Timestamp(_, _)- | Utf8- | LargeUtf8- | Utf8View- | Binary- | LargeBinary- | BinaryView- | Duration(_)- | Struct(_)- )+ let data_type = args.return_field.data_type();+ data_type.is_primitive()+ || matches!(+ data_type,+ Utf8 | LargeUtf8+ | Utf8View+ | Binary+ | LargeBinary+ | BinaryView+ | Duration(_)+ | Struct(_)+ )}fn create_groups_accumulator(@@ -283,84 +256,31 @@ impl AggregateUDFImpl for Max {args: AccumulatorArgs,) -> Result<Box<dyn GroupsAccumulator>> {use DataType::*;- use TimeUnit::*;let data_type = args.return_field.data_type();+match data_type {- Int8 => primitive_max_accumulator!(data_type, i8, Int8Type),- Int16 => primitive_max_accumulator!(data_type, i16, Int16Type),- Int32 => primitive_max_accumulator!(data_type, i32, Int32Type),- Int64 => primitive_max_accumulator!(data_type, i64, Int64Type),- UInt8 => primitive_max_accumulator!(data_type, u8, UInt8Type),- UInt16 => primitive_max_accumulator!(data_type, u16, UInt16Type),- UInt32 => primitive_max_accumulator!(data_type, u32, UInt32Type),- UInt64 => primitive_max_accumulator!(data_type, u64, UInt64Type),Float16 => {- primitive_max_accumulator!(data_type, f16, Float16Type, total, u16)+ primitive_max_accumulator!(Float16Type, data_type, total, u16)}Float32 => {- primitive_max_accumulator!(data_type, f32, Float32Type, total, u32)+ primitive_max_accumulator!(Float32Type, data_type, total, u32)}Float64 => {- primitive_max_accumulator!(data_type, f64, Float64Type, total, u64)- }- Date32 => primitive_max_accumulator!(data_type, i32, Date32Type),- Date64 => primitive_max_accumulator!(data_type, i64, Date64Type),- Time32(Second) => {- primitive_max_accumulator!(data_type, i32, Time32SecondType)- }- Time32(Millisecond) => {- primitive_max_accumulator!(data_type, i32, Time32MillisecondType)- }- Time64(Microsecond) => {- primitive_max_accumulator!(data_type, i64, Time64MicrosecondType)- }- Time64(Nanosecond) => {- primitive_max_accumulator!(data_type, i64, Time64NanosecondType)- }- Timestamp(Second, _) => {- primitive_max_accumulator!(data_type, i64, TimestampSecondType)- }- Timestamp(Millisecond, _) => {- primitive_max_accumulator!(data_type, i64, TimestampMillisecondType)- }- Timestamp(Microsecond, _) => {- primitive_max_accumulator!(data_type, i64, TimestampMicrosecondType)- }- Timestamp(Nanosecond, _) => {- primitive_max_accumulator!(data_type, i64, TimestampNanosecondType)- }- Duration(Second) => {- primitive_max_accumulator!(data_type, i64, DurationSecondType)- }- Duration(Millisecond) => {- primitive_max_accumulator!(data_type, i64, DurationMillisecondType)+ primitive_max_accumulator!(Float64Type, data_type, total, u64)}- Duration(Microsecond) => {- primitive_max_accumulator!(data_type, i64, DurationMicrosecondType)- }- Duration(Nanosecond) => {- primitive_max_accumulator!(data_type, i64, DurationNanosecondType)- }- Decimal32(_, _) => {- primitive_max_accumulator!(data_type, i32, Decimal32Type)- }- Decimal64(_, _) => {- primitive_max_accumulator!(data_type, i64, Decimal64Type)- }- Decimal128(_, _) => {- primitive_max_accumulator!(data_type, i128, Decimal128Type)- }- Decimal256(_, _) => {- primitive_max_accumulator!(data_type, i256, Decimal256Type)- }- Utf8 | LargeUtf8 | Utf8View | Binary | LargeBinary | BinaryView => {- Ok(Box::new(MinMaxBytesAccumulator::new_max(data_type.clone())))+ _ => {+ downcast_primitive! {+ data_type => (primitive_max_accumulator, data_type),+ Utf8 | LargeUtf8 | Utf8View | Binary | LargeBinary | BinaryView => {+ Ok(Box::new(MinMaxBytesAccumulator::new_max(data_type.clone())))+ }+ Struct(_) => Ok(Box::new(MinMaxStructAccumulator::new_max(+ data_type.clone(),+ ))),+ // This is only reached if groups_accumulator_supported is out of sync+ _ => internal_err!("GroupsAccumulator not supported for max({})", data_type),+ }}- Struct(_) => Ok(Box::new(MinMaxStructAccumulator::new_max(- data_type.clone(),- ))),- // This is only reached if groups_accumulator_supported is out of sync- _ => internal_err!("GroupsAccumulator not supported for max({})", data_type),}}@@ -545,36 +465,19 @@ impl AggregateUDFImpl for Min {fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {use DataType::*;- matches!(- args.return_field.data_type(),- Int8 | Int16- | Int32- | Int64- | UInt8- | UInt16- | UInt32- | UInt64- | Float16- | Float32- | Float64- | Decimal32(_, _)- | Decimal64(_, _)- | Decimal128(_, _)- | Decimal256(_, _)- | Date32- | Date64- | Time32(_)- | Time64(_)- | Timestamp(_, _)- | Utf8- | LargeUtf8- | Utf8View- | Binary- | LargeBinary- | BinaryView- | Duration(_)- | Struct(_)- )++ let data_type = args.return_field.data_type();+ data_type.is_primitive()+ || matches!(+ data_type,+ Utf8 | LargeUtf8+ | Utf8View+ | Binary+ | LargeBinary+ | BinaryView+ | Duration(_)+ | Struct(_)+ )}fn create_groups_accumulator(@@ -582,84 +485,31 @@ impl AggregateUDFImpl for Min {args: AccumulatorArgs,) -> Result<Box<dyn GroupsAccumulator>> {use DataType::*;- use TimeUnit::*;let data_type = args.return_field.data_type();+match data_type {- Int8 => primitive_min_accumulator!(data_type, i8, Int8Type),- Int16 => primitive_min_accumulator!(data_type, i16, Int16Type),- Int32 => primitive_min_accumulator!(data_type, i32, Int32Type),- Int64 => primitive_min_accumulator!(data_type, i64, Int64Type),- UInt8 => primitive_min_accumulator!(data_type, u8, UInt8Type),- UInt16 => primitive_min_accumulator!(data_type, u16, UInt16Type),- UInt32 => primitive_min_accumulator!(data_type, u32, UInt32Type),- UInt64 => primitive_min_accumulator!(data_type, u64, UInt64Type),Float16 => {- primitive_min_accumulator!(data_type, f16, Float16Type, total, u16)+ primitive_min_accumulator!(Float16Type, data_type, total, u16)}Float32 => {- primitive_min_accumulator!(data_type, f32, Float32Type, total, u32)+ primitive_min_accumulator!(Float32Type, data_type, total, u32)}Float64 => {- primitive_min_accumulator!(data_type, f64, Float64Type, total, u64)- }- Date32 => primitive_min_accumulator!(data_type, i32, Date32Type),- Date64 => primitive_min_accumulator!(data_type, i64, Date64Type),- Time32(Second) => {- primitive_min_accumulator!(data_type, i32, Time32SecondType)- }- Time32(Millisecond) => {- primitive_min_accumulator!(data_type, i32, Time32MillisecondType)- }- Time64(Microsecond) => {- primitive_min_accumulator!(data_type, i64, Time64MicrosecondType)+ primitive_min_accumulator!(Float64Type, data_type, total, u64)}- Time64(Nanosecond) => {- primitive_min_accumulator!(data_type, i64, Time64NanosecondType)- }- Timestamp(Second, _) => {- primitive_min_accumulator!(data_type, i64, TimestampSecondType)- }- Timestamp(Millisecond, _) => {- primitive_min_accumulator!(data_type, i64, TimestampMillisecondType)- }- Timestamp(Microsecond, _) => {- primitive_min_accumulator!(data_type, i64, TimestampMicrosecondType)- }- Timestamp(Nanosecond, _) => {- primitive_min_accumulator!(data_type, i64, TimestampNanosecondType)- }- Duration(Second) => {- primitive_min_accumulator!(data_type, i64, DurationSecondType)- }- Duration(Millisecond) => {- primitive_min_accumulator!(data_type, i64, DurationMillisecondType)- }- Duration(Microsecond) => {- primitive_min_accumulator!(data_type, i64, DurationMicrosecondType)- }- Duration(Nanosecond) => {- primitive_min_accumulator!(data_type, i64, DurationNanosecondType)- }- Decimal32(_, _) => {- primitive_min_accumulator!(data_type, i32, Decimal32Type)- }- Decimal64(_, _) => {- primitive_min_accumulator!(data_type, i64, Decimal64Type)- }- Decimal128(_, _) => {- primitive_min_accumulator!(data_type, i128, Decimal128Type)- }- Decimal256(_, _) => {- primitive_min_accumulator!(data_type, i256, Decimal256Type)- }- Utf8 | LargeUtf8 | Utf8View | Binary | LargeBinary | BinaryView => {- Ok(Box::new(MinMaxBytesAccumulator::new_min(data_type.clone())))+ _ => {+ downcast_primitive! {+ data_type => (primitive_min_accumulator, data_type),+ Utf8 | LargeUtf8 | Utf8View | Binary | LargeBinary | BinaryView => {+ Ok(Box::new(MinMaxBytesAccumulator::new_min(data_type.clone())))+ }+ Struct(_) => Ok(Box::new(MinMaxStructAccumulator::new_min(+ data_type.clone(),+ ))),+ // This is only reached if groups_accumulator_supported is out of sync+ _ => internal_err!("GroupsAccumulator not supported for min({})", data_type),+ }}- Struct(_) => Ok(Box::new(MinMaxStructAccumulator::new_min(- data_type.clone(),- ))),- // This is only reached if groups_accumulator_supported is out of sync- _ => internal_err!("GroupsAccumulator not supported for min({})", data_type),}}@@ -1052,8 +902,8 @@ mod tests {PrimitiveArray, StringArray,},datatypes::{- ArrowDictionaryKeyType, IntervalDayTimeType, IntervalMonthDayNanoType,- IntervalUnit, IntervalYearMonthType,+ ArrowDictionaryKeyType, Field, Float32Type, IntervalDayTimeType,+ IntervalMonthDayNanoType, IntervalUnit, IntervalYearMonthType, Schema,},};use datafusion_expr::EmitTo;@@ -1063,12 +913,12 @@ mod tests {fn grouped_float_min_max_total_order() -> Result<()> {fn grouped_float32_min() -> Result<Box<dyn GroupsAccumulator>> {let data_type = &DataType::Float32;- primitive_min_accumulator!(data_type, f32, Float32Type, total, u32)+ primitive_min_accumulator!(Float32Type, data_type, total, u32)}fn grouped_float32_max() -> Result<Box<dyn GroupsAccumulator>> {let data_type = &DataType::Float32;- primitive_max_accumulator!(data_type, f32, Float32Type, total, u32)+ primitive_max_accumulator!(Float32Type, data_type, total, u32)}fn evaluate_grouped_float32(@@ -1214,6 +1064,180 @@ mod tests {);}+ /// Runs `udf` through its [`GroupsAccumulator`], feeding `values` to two+ /// accumulators (first and second half of the rows) and merging the state+ /// of the first into the second, so that `update_batch`, `state` and+ /// `merge_batch` are all exercised.+ fn evaluate_grouped(+ udf: &dyn AggregateUDFImpl,+ values: &ArrayRef,+ group_indices: &[usize],+ total_num_groups: usize,+ ) -> Result<ArrayRef> {+ let data_type = values.data_type().clone();+ let schema = Schema::new(vec![Field::new("v", data_type.clone(), true)]);+ let expr: Arc<dyn datafusion_physical_expr::PhysicalExpr> =+ Arc::new(expressions::Column::new("v", 0));+ let expr_field = Arc::new(Field::new("v", data_type.clone(), true));+ let args = || AccumulatorArgs {+ schema: &schema,+ expr_fields: std::slice::from_ref(&expr_field),+ exprs: std::slice::from_ref(&expr),+ is_distinct: false,+ name: udf.name(),+ ignore_nulls: false,+ is_reversed: false,+ return_field: Arc::new(Field::new(udf.name(), data_type.clone(), true)),+ order_bys: &[],+ };++ let mid = values.len() / 2;+ let mut first = udf.create_groups_accumulator(args())?;+ first.update_batch(+ &[values.slice(0, mid)],+ &group_indices[..mid],+ None,+ total_num_groups,+ )?;++ let mut second = udf.create_groups_accumulator(args())?;+ second.update_batch(+ &[values.slice(mid, values.len() - mid)],+ &group_indices[mid..],+ None,+ total_num_groups,+ )?;++ let state = first.state(EmitTo::All)?;+ let merge_group_indices: Vec<usize> = (0..total_num_groups).collect();+ second.merge_batch(&state, &merge_group_indices, total_num_groups)?;++ let result = second.evaluate(EmitTo::All)?;+ assert_eq!(result.data_type(), &data_type);+ Ok(result)+ }++ #[test]+ fn interval_min_max_groups_accumulator() -> Result<()> {+ // Group 0 and 1 get values from both halves of the input, group 2 only+ // sees nulls and group 3 never receives any row.+ let group_indices = [0, 1, 2, 0, 1, 0, 1, 2];+ let total_num_groups = 4;++ let cases: Vec<(ArrayRef, ArrayRef, ArrayRef)> = vec![+ (+ Arc::new(IntervalYearMonthArray::from(vec![+ Some(IntervalYearMonthType::make_value(0, 1)),+ Some(IntervalYearMonthType::make_value(5, 34)),+ None,+ Some(IntervalYearMonthType::make_value(-2, 4)),+ Some(IntervalYearMonthType::make_value(7, -4)),+ Some(IntervalYearMonthType::make_value(0, 1)),+ None,+ None,+ ])),+ // min+ Arc::new(IntervalYearMonthArray::from(vec![+ Some(IntervalYearMonthType::make_value(-2, 4)),+ Some(IntervalYearMonthType::make_value(7, -4)),+ None,+ None,+ ])),+ // max+ Arc::new(IntervalYearMonthArray::from(vec![+ Some(IntervalYearMonthType::make_value(0, 1)),+ Some(IntervalYearMonthType::make_value(5, 34)),+ None,+ None,+ ])),+ ),+ (+ Arc::new(IntervalDayTimeArray::from(vec![+ Some(IntervalDayTimeType::make_value(0, 0)),+ Some(IntervalDayTimeType::make_value(5, 454000)),+ None,+ Some(IntervalDayTimeType::make_value(-34, 0)),+ Some(IntervalDayTimeType::make_value(7, -4000)),+ Some(IntervalDayTimeType::make_value(1, 0)),+ Some(IntervalDayTimeType::make_value(5, 453999)),+ None,+ ])),+ // min+ Arc::new(IntervalDayTimeArray::from(vec![+ Some(IntervalDayTimeType::make_value(-34, 0)),+ Some(IntervalDayTimeType::make_value(5, 453999)),+ None,+ None,+ ])),+ // max+ Arc::new(IntervalDayTimeArray::from(vec![+ Some(IntervalDayTimeType::make_value(1, 0)),+ Some(IntervalDayTimeType::make_value(7, -4000)),+ None,+ None,+ ])),+ ),+ (+ Arc::new(IntervalMonthDayNanoArray::from(vec![+ Some(IntervalMonthDayNanoType::make_value(1, 0, 0)),+ Some(IntervalMonthDayNanoType::make_value(+ 344,+ 34,+ -43_000_000_000,+ )),+ None,+ Some(IntervalMonthDayNanoType::make_value(+ -593,+ -33,+ 13_000_000_000,+ )),+ Some(IntervalMonthDayNanoType::make_value(5, 2, 493_000_000_000)),+ Some(IntervalMonthDayNanoType::make_value(1, 0, 1)),+ Some(IntervalMonthDayNanoType::make_value(+ 344,+ 34,+ -42_000_000_000,+ )),+ None,+ ])),+ // min+ Arc::new(IntervalMonthDayNanoArray::from(vec![+ Some(IntervalMonthDayNanoType::make_value(+ -593,+ -33,+ 13_000_000_000,+ )),+ Some(IntervalMonthDayNanoType::make_value(5, 2, 493_000_000_000)),+ None,+ None,+ ])),+ // max+ Arc::new(IntervalMonthDayNanoArray::from(vec![+ Some(IntervalMonthDayNanoType::make_value(1, 0, 1)),+ Some(IntervalMonthDayNanoType::make_value(+ 344,+ 34,+ -42_000_000_000,+ )),+ None,+ None,+ ])),+ ),+ ];++ for (values, expected_min, expected_max) in cases {+ let min =+ evaluate_grouped(&Min::new(), &values, &group_indices, total_num_groups)?;+ assert_eq!(&min, &expected_min, "min({})", values.data_type());++ let max =+ evaluate_grouped(&Max::new(), &values, &group_indices, total_num_groups)?;+ assert_eq!(&max, &expected_max, "max({})", values.data_type());+ }++ Ok(())+ }+#[test]fn float_min_max_with_nans() {let pos_nan = f32::NAN;diff --git a/datafusion/physical-plan/src/aggregates/group_values/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/mod.rsindex 80e6b335e3..e03db633e6 100644--- a/datafusion/physical-plan/src/aggregates/group_values/mod.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/mod.rs@@ -17,13 +17,8 @@//! [`GroupValues`] trait for storing and interning group keys-use arrow::array::types::{- Date32Type, Date64Type, Decimal128Type, Time32MillisecondType, Time32SecondType,- Time64MicrosecondType, Time64NanosecondType, TimestampMicrosecondType,- TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,-};use arrow::array::{ArrayRef, downcast_primitive};-use arrow::datatypes::{DataType, SchemaRef, TimeUnit};+use arrow::datatypes::{DataType, SchemaRef};use datafusion_common::{Result, not_impl_err};use datafusion_expr::{EmitTo, GroupSelection};@@ -171,35 +166,6 @@ pub fn new_group_values(downcast_primitive! {d => (downcast_helper, d),- _ => {}- }-- match d {- DataType::Date32 => {- downcast_helper!(Date32Type, d);- }- DataType::Date64 => {- downcast_helper!(Date64Type, d);- }- DataType::Time32(t) => match t {- TimeUnit::Second => downcast_helper!(Time32SecondType, d),- TimeUnit::Millisecond => downcast_helper!(Time32MillisecondType, d),- _ => {}- },- DataType::Time64(t) => match t {- TimeUnit::Microsecond => downcast_helper!(Time64MicrosecondType, d),- TimeUnit::Nanosecond => downcast_helper!(Time64NanosecondType, d),- _ => {}- },- DataType::Timestamp(t, _tz) => match t {- TimeUnit::Second => downcast_helper!(TimestampSecondType, d),- TimeUnit::Millisecond => downcast_helper!(TimestampMillisecondType, d),- TimeUnit::Microsecond => downcast_helper!(TimestampMicrosecondType, d),- TimeUnit::Nanosecond => downcast_helper!(TimestampNanosecondType, d),- },- DataType::Decimal128(_, _) => {- downcast_helper!(Decimal128Type, d);- }DataType::Utf8 => {return Ok(Box::new(GroupValuesBytes::<i32>::new(OutputType::Utf8)));}diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rsindex 098b49dd4f..07b825e95a 100644--- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs@@ -36,17 +36,12 @@ use crate::aggregates::group_values::multi_group_by::{fixed_size_binary::FixedSizeBinaryGroupValueBuilder,primitive::PrimitiveGroupValueBuilder, row_backed::RowsGroupColumn,};-use arrow::array::{Array, ArrayRef, BooleanBufferBuilder, new_empty_array};+use arrow::array::{+ Array, ArrayRef, BooleanBufferBuilder, downcast_primitive, new_empty_array,+};use arrow::datatypes::{- BinaryViewType, DataType, Date32Type, Date64Type, Decimal32Type, Decimal64Type,- Decimal128Type, Decimal256Type, DurationMicrosecondType, DurationMillisecondType,- DurationNanosecondType, DurationSecondType, Field, Float16Type, Float32Type,- Float64Type, Int8Type, Int16Type, Int32Type, Int64Type, IntervalDayTimeType,- IntervalMonthDayNanoType, IntervalUnit, IntervalYearMonthType, Schema, SchemaRef,- StringViewType, Time32MillisecondType, Time32SecondType, Time64MicrosecondType,- Time64NanosecondType, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,- TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type,- UInt64Type,+ BinaryViewType, DataType, Field, Int8Type, Int16Type, Int32Type, Int64Type, Schema,+ SchemaRef, StringViewType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,};use datafusion_common::hash_utils::RandomState;use datafusion_common::hash_utils::create_hashes;@@ -913,11 +908,11 @@ impl<const STREAMING: bool> GroupValuesColumn<STREAMING> {/// instantiates a [`PrimitiveGroupValueBuilder`] and pushes it into $v////// Arguments:+/// `$t`: the primitive type of the builder/// `$v`: the vector to push the new builder into/// `$nullable`: whether the input can contains nulls-/// `$t`: the primitive type of the buildermacro_rules! instantiate_primitive {- ($nullable:expr, $t:ty, $data_type:ident) => {{+ ($t:ty, $data_type:ident, $nullable:expr) => {{let builder: Box<dyn GroupColumn> = if $nullable {Box::new(PrimitiveGroupValueBuilder::<$t, true>::new($data_type.to_owned(),@@ -959,95 +954,10 @@ fn group_column_supported_type(data_type: &DataType) -> bool {fn make_group_column(field: &Field) -> Result<Box<dyn GroupColumn>> {let nullable = field.is_nullable();let data_type = field.data_type();- let builder: Option<Box<dyn GroupColumn>> = match data_type {- DataType::Int8 => instantiate_primitive!(nullable, Int8Type, data_type),- DataType::Int16 => instantiate_primitive!(nullable, Int16Type, data_type),- DataType::Int32 => instantiate_primitive!(nullable, Int32Type, data_type),- DataType::Int64 => instantiate_primitive!(nullable, Int64Type, data_type),- DataType::UInt8 => instantiate_primitive!(nullable, UInt8Type, data_type),- DataType::UInt16 => instantiate_primitive!(nullable, UInt16Type, data_type),- DataType::UInt32 => instantiate_primitive!(nullable, UInt32Type, data_type),- DataType::UInt64 => instantiate_primitive!(nullable, UInt64Type, data_type),- DataType::Float16 => instantiate_primitive!(nullable, Float16Type, data_type),- DataType::Float32 => instantiate_primitive!(nullable, Float32Type, data_type),- DataType::Float64 => instantiate_primitive!(nullable, Float64Type, data_type),- DataType::Date32 => instantiate_primitive!(nullable, Date32Type, data_type),- DataType::Date64 => instantiate_primitive!(nullable, Date64Type, data_type),- DataType::Time32(t) => match t {- TimeUnit::Second => {- instantiate_primitive!(nullable, Time32SecondType, data_type)- }- TimeUnit::Millisecond => {- instantiate_primitive!(nullable, Time32MillisecondType, data_type)- }- // Time32 with Microsecond / Nanosecond is not a valid Arrow type- // combination; reject explicitly.- _ => None,- },- DataType::Time64(t) => match t {- TimeUnit::Microsecond => {- instantiate_primitive!(nullable, Time64MicrosecondType, data_type)- }- TimeUnit::Nanosecond => {- instantiate_primitive!(nullable, Time64NanosecondType, data_type)- }- // Time64 with Second / Millisecond is not a valid Arrow type- // combination; reject explicitly.- _ => None,- },- DataType::Timestamp(t, _) => match t {- TimeUnit::Second => {- instantiate_primitive!(nullable, TimestampSecondType, data_type)- }- TimeUnit::Millisecond => {- instantiate_primitive!(nullable, TimestampMillisecondType, data_type)- }- TimeUnit::Microsecond => {- instantiate_primitive!(nullable, TimestampMicrosecondType, data_type)- }- TimeUnit::Nanosecond => {- instantiate_primitive!(nullable, TimestampNanosecondType, data_type)- }- },- DataType::Duration(t) => match t {- TimeUnit::Second => {- instantiate_primitive!(nullable, DurationSecondType, data_type)- }- TimeUnit::Millisecond => {- instantiate_primitive!(nullable, DurationMillisecondType, data_type)- }- TimeUnit::Microsecond => {- instantiate_primitive!(nullable, DurationMicrosecondType, data_type)- }- TimeUnit::Nanosecond => {- instantiate_primitive!(nullable, DurationNanosecondType, data_type)- }- },- // `IntervalUnit` has exactly three variants, so this match is exhaustive- // with no fallback arm (unlike Time32 / Time64).- DataType::Interval(u) => match u {- IntervalUnit::YearMonth => {- instantiate_primitive!(nullable, IntervalYearMonthType, data_type)- }- IntervalUnit::DayTime => {- instantiate_primitive!(nullable, IntervalDayTimeType, data_type)- }- IntervalUnit::MonthDayNano => {- instantiate_primitive!(nullable, IntervalMonthDayNanoType, data_type)- }- },- DataType::Decimal32(_, _) => {- instantiate_primitive!(nullable, Decimal32Type, data_type)- }- DataType::Decimal64(_, _) => {- instantiate_primitive!(nullable, Decimal64Type, data_type)- }- DataType::Decimal128(_, _) => {- instantiate_primitive!(nullable, Decimal128Type, data_type)- }- DataType::Decimal256(_, _) => {- instantiate_primitive!(nullable, Decimal256Type, data_type)- }++ let builder: Option<Box<dyn GroupColumn>> = downcast_primitive! {+ data_type => (instantiate_primitive, data_type, nullable),+DataType::Utf8 => Some(Box::new(ByteGroupValueBuilder::<i32>::new(OutputType::Utf8,))),