Apache DataFusion SQL Query Engine
refactor: share aggregate return field metadata handling (#27193)
RIchard Baah3b0d0c8e1960
diff --git a/datafusion/expr/src/lib.rs b/datafusion/expr/src/lib.rsindex 92eb0d1e9a..a6dd1e7bca 100644--- a/datafusion/expr/src/lib.rs+++ b/datafusion/expr/src/lib.rs@@ -136,6 +136,7 @@ pub use udaf::{StatisticsArgs, UdafDisplayNameBuilder, UdafHumanDisplayBuilder,UdafSchemaNameBuilder, UdafWindowFunctionDisplayNameBuilder,UdafWindowFunctionSchemaNameBuilder, udaf_default_return_field,+ udaf_return_field_preserving_arg_metadata,};#[expect(deprecated)]pub use udaf::{diff --git a/datafusion/expr/src/udaf.rs b/datafusion/expr/src/udaf.rsindex 0cfba5786a..a520e18bd4 100644--- a/datafusion/expr/src/udaf.rs+++ b/datafusion/expr/src/udaf.rs@@ -13,7 +13,10 @@ use std::vec;use arrow::datatypes::{DataType, Field, FieldRef};-use datafusion_common::{Result, ScalarValue, Statistics, exec_err, not_impl_err};+use datafusion_common::{+ Result, ScalarValue, Statistics, exec_err, internal_err, metadata::FieldMetadata,+ not_impl_err,+};use datafusion_expr_common::dyn_eq::{DynEq, DynHash};use datafusion_expr_common::operator::Operator;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;@@ -1509,6 +1512,23 @@ pub fn udaf_default_return_field<F: AggregateUDFImpl + ?Sized>()))}+/// Returns the default field for `func` with metadata from its first argument.+///+/// This is useful for aggregate functions that return a value from their first+/// argument, such as `min`, `max`, and `first_value`.+pub fn udaf_return_field_preserving_arg_metadata<F: AggregateUDFImpl + ?Sized>(+ func: &F,+ arg_fields: &[FieldRef],+) -> Result<FieldRef> {+ let Some(arg_field) = arg_fields.first() else {+ return internal_err!(+ "udaf_return_field_preserving_arg_metadata requires at least one argument"+ );+ };+ let field = udaf_default_return_field(func, arg_fields)?;+ Ok(FieldMetadata::from(arg_field.as_ref()).add_to_field_ref(field))+}+pub enum ReversedUDAF {/// The expression is the same as the original expression, like SUM, COUNTIdentical,diff --git a/datafusion/functions-aggregate/src/any_value.rs b/datafusion/functions-aggregate/src/any_value.rsindex 2f49a1f0a2..2ea9e35594 100644--- a/datafusion/functions-aggregate/src/any_value.rs+++ b/datafusion/functions-aggregate/src/any_value.rs@@ -6,14 +6,14 @@use std::fmt::Debug;use std::hash::Hash;-use std::sync::Arc;use arrow::datatypes::{DataType, Field, FieldRef};-use datafusion_common::{Result, not_impl_err};+use datafusion_common::Result;use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};use datafusion_expr::{Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,+ udaf_return_field_preserving_arg_metadata,};use datafusion_macros::user_doc;@@ -69,15 +69,12 @@ impl AggregateUDFImpl for AnyValue {&self.signature}- fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {- not_impl_err!("Not called because return_field is implemented")+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {+ Ok(arg_types[0].clone())}fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {- Ok(Arc::new(- Field::new(self.name(), arg_fields[0].data_type().clone(), true)- .with_metadata(arg_fields[0].metadata().clone()),- ))+ udaf_return_field_preserving_arg_metadata(self, arg_fields)}fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {diff --git a/datafusion/functions-aggregate/src/first_last.rs b/datafusion/functions-aggregate/src/first_last.rsindex 40f961157e..8b5b34dea2 100644--- a/datafusion/functions-aggregate/src/first_last.rs+++ b/datafusion/functions-aggregate/src/first_last.rs@@ -24,7 +24,6 @@ use datafusion_common::cast::as_boolean_array;use datafusion_common::utils::{compare_rows, extract_row_at_idx_to_buf, get_row_at_idx};use datafusion_common::{DataFusionError, Result, ScalarValue, arrow_datafusion_err, internal_err,- not_impl_err,};use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};use datafusion_expr::utils::{@@ -33,6 +32,7 @@ use datafusion_expr::utils::{use datafusion_expr::{Accumulator, AggregateUDFImpl, Documentation, EmitTo, Expr, ExprFunctionExt,GroupsAccumulator, ReversedUDAF, Signature, SortExpr, Volatility,+ udaf_return_field_preserving_arg_metadata,};use datafusion_functions_aggregate_common::utils::get_sort_options;use datafusion_macros::user_doc;@@ -284,20 +284,12 @@ impl AggregateUDFImpl for FirstValue {&self.signature}- fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {- not_impl_err!("Not called because the return_field_from_args is implemented")+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {+ Ok(arg_types[0].clone())}fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {- // Preserve metadata from the first argument field- Ok(Arc::new(- Field::new(- self.name(),- arg_fields[0].data_type().clone(),- true, // always nullable, there may be no rows- )- .with_metadata(arg_fields[0].metadata().clone()),- ))+ udaf_return_field_preserving_arg_metadata(self, arg_fields)}fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {@@ -1342,20 +1334,12 @@ impl AggregateUDFImpl for LastValue {&self.signature}- fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {- not_impl_err!("Not called because the return_field_from_args is implemented")+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {+ Ok(arg_types[0].clone())}fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {- // Preserve metadata from the first argument field- Ok(Arc::new(- Field::new(- self.name(),- arg_fields[0].data_type().clone(),- true, // always nullable, there may be no rows- )- .with_metadata(arg_fields[0].metadata().clone()),- ))+ udaf_return_field_preserving_arg_metadata(self, arg_fields)}fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {diff --git a/datafusion/functions-aggregate/src/min_max.rs b/datafusion/functions-aggregate/src/min_max.rsindex 88fe168b6f..2af3bbfd5d 100644--- a/datafusion/functions-aggregate/src/min_max.rs+++ b/datafusion/functions-aggregate/src/min_max.rs@@ -11,7 +11,6 @@ mod min_max_struct;use arrow::array::{ArrayRef, ArrowPrimitiveType, downcast_primitive};use arrow::datatypes::{DataType, FieldRef, Float16Type, Float32Type, Float64Type};use datafusion_common::float_semantics::{float_gt, float_lt};-use datafusion_common::metadata::FieldMetadata;use datafusion_common::stats::Precision;use datafusion_common::{ColumnStatistics, Result, exec_err, internal_err};use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;@@ -25,7 +24,7 @@ use datafusion_common::ScalarValue;use datafusion_expr::DistinctHandling;use datafusion_expr::{Accumulator, AggregateUDFImpl, Documentation, SetMonotonicity, Signature, Volatility,- function::AccumulatorArgs, udaf_default_return_field,+ function::AccumulatorArgs, udaf_return_field_preserving_arg_metadata,};use datafusion_expr::{GroupsAccumulator, StatisticsArgs};use datafusion_macros::user_doc;@@ -208,16 +207,6 @@ impl FromColumnStatistics for Max {}}-/// Returns the field of the result of `min` or `max`. The result has the type of the argument,-/// so it keeps the metadata of the argument field, such as an extension type.-fn min_max_return_field(- func: &impl AggregateUDFImpl,- arg_fields: &[FieldRef],-) -> Result<FieldRef> {- let field = udaf_default_return_field(func, arg_fields)?;- Ok(FieldMetadata::from(arg_fields[0].as_ref()).add_to_field_ref(field))-}-impl AggregateUDFImpl for Max {fn name(&self) -> &str {"max"@@ -232,7 +221,7 @@ impl AggregateUDFImpl for Max {}fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {- min_max_return_field(self, arg_fields)+ udaf_return_field_preserving_arg_metadata(self, arg_fields)}fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {@@ -464,7 +453,7 @@ impl AggregateUDFImpl for Min {}fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {- min_max_return_field(self, arg_fields)+ udaf_return_field_preserving_arg_metadata(self, arg_fields)}fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {diff --git a/datafusion/functions-aggregate/src/nth_value.rs b/datafusion/functions-aggregate/src/nth_value.rsindex 045f2891cf..674cfa1f6d 100644--- a/datafusion/functions-aggregate/src/nth_value.rs+++ b/datafusion/functions-aggregate/src/nth_value.rs@@ -22,7 +22,7 @@ use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};use datafusion_expr::utils::format_state_name;use datafusion_expr::{Accumulator, AggregateUDFImpl, Documentation, ExprFunctionExt, ReversedUDAF,- Signature, SortExpr, Volatility, lit,+ Signature, SortExpr, Volatility, lit, udaf_return_field_preserving_arg_metadata,};use datafusion_functions_aggregate_common::merge_arrays::merge_ordered_arrays;use datafusion_functions_aggregate_common::utils::ordering_fields;@@ -114,6 +114,10 @@ impl AggregateUDFImpl for NthValueAgg {Ok(arg_types[0].clone())}+ fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {+ udaf_return_field_preserving_arg_metadata(self, arg_fields)+ }+fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {let n = match acc_args.exprs[1].downcast_ref::<Literal>()diff --git a/datafusion/sqllogictest/test_files/metadata.slt b/datafusion/sqllogictest/test_files/metadata.sltindex 045e8b39b2..b86a6ce3fc 100644--- a/datafusion/sqllogictest/test_files/metadata.slt+++ b/datafusion/sqllogictest/test_files/metadata.slt@@ -346,6 +346,20 @@ from table_with_metadata;----3 the id field+# Regression test: nth_value aggregate should preserve metadata+query IT+select nth_value(id, 2 order by id asc nulls last), arrow_metadata(nth_value(id, 2 order by id asc nulls last), 'metadata_key')+from table_with_metadata;+----+3 the id field++# Regression test: any_value should preserve metadata+query T+select arrow_metadata(any_value(id), 'metadata_key')+from table_with_metadata;+----+the id field+# Regression test: max and min should preserve metadataquery ITITselect