Skip to content

Apache DataFusion SQL Query Engine

refactor: share aggregate return field metadata handling (#27193)

RIchard Baah3b0d0c8e1960
diff --git a/datafusion/expr/src/lib.rs b/datafusion/expr/src/lib.rs
index 92eb0d1e9a..a6dd1e7bca 100644
--- a/datafusion/expr/src/lib.rs
+++ b/datafusion/expr/src/lib.rs
@@ -136,6 +136,7 @@ pub use udaf::{
StatisticsArgs, UdafDisplayNameBuilder, UdafHumanDisplayBuilder,
UdafSchemaNameBuilder, UdafWindowFunctionDisplayNameBuilder,
UdafWindowFunctionSchemaNameBuilder, udaf_default_return_field,
+ udaf_return_field_preserving_arg_metadata,
};
#[expect(deprecated)]
pub use udaf::{
diff --git a/datafusion/expr/src/udaf.rs b/datafusion/expr/src/udaf.rs
index 0cfba5786a..a520e18bd4 100644
--- a/datafusion/expr/src/udaf.rs
+++ b/datafusion/expr/src/udaf.rs
@@ -13,7 +13,10 @@ use std::vec;
use arrow::datatypes::{DataType, Field, FieldRef};
-use datafusion_common::{Result, ScalarValue, Statistics, exec_err, not_impl_err};
+use datafusion_common::{
+ Result, ScalarValue, Statistics, exec_err, internal_err, metadata::FieldMetadata,
+ not_impl_err,
+};
use datafusion_expr_common::dyn_eq::{DynEq, DynHash};
use datafusion_expr_common::operator::Operator;
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
@@ -1509,6 +1512,23 @@ pub fn udaf_default_return_field<F: AggregateUDFImpl + ?Sized>(
)))
}
+/// Returns the default field for `func` with metadata from its first argument.
+///
+/// This is useful for aggregate functions that return a value from their first
+/// argument, such as `min`, `max`, and `first_value`.
+pub fn udaf_return_field_preserving_arg_metadata<F: AggregateUDFImpl + ?Sized>(
+ func: &F,
+ arg_fields: &[FieldRef],
+) -> Result<FieldRef> {
+ let Some(arg_field) = arg_fields.first() else {
+ return internal_err!(
+ "udaf_return_field_preserving_arg_metadata requires at least one argument"
+ );
+ };
+ let field = udaf_default_return_field(func, arg_fields)?;
+ Ok(FieldMetadata::from(arg_field.as_ref()).add_to_field_ref(field))
+}
+
pub enum ReversedUDAF {
/// The expression is the same as the original expression, like SUM, COUNT
Identical,
diff --git a/datafusion/functions-aggregate/src/any_value.rs b/datafusion/functions-aggregate/src/any_value.rs
index 2f49a1f0a2..2ea9e35594 100644
--- a/datafusion/functions-aggregate/src/any_value.rs
+++ b/datafusion/functions-aggregate/src/any_value.rs
@@ -6,14 +6,14 @@
use std::fmt::Debug;
use std::hash::Hash;
-use std::sync::Arc;
use arrow::datatypes::{DataType, Field, FieldRef};
-use datafusion_common::{Result, not_impl_err};
+use datafusion_common::Result;
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,
+ udaf_return_field_preserving_arg_metadata,
};
use datafusion_macros::user_doc;
@@ -69,15 +69,12 @@ impl AggregateUDFImpl for AnyValue {
&self.signature
}
- fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
- not_impl_err!("Not called because return_field is implemented")
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ Ok(arg_types[0].clone())
}
fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
- Ok(Arc::new(
- Field::new(self.name(), arg_fields[0].data_type().clone(), true)
- .with_metadata(arg_fields[0].metadata().clone()),
- ))
+ udaf_return_field_preserving_arg_metadata(self, arg_fields)
}
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
diff --git a/datafusion/functions-aggregate/src/first_last.rs b/datafusion/functions-aggregate/src/first_last.rs
index 40f961157e..8b5b34dea2 100644
--- a/datafusion/functions-aggregate/src/first_last.rs
+++ b/datafusion/functions-aggregate/src/first_last.rs
@@ -24,7 +24,6 @@ use datafusion_common::cast::as_boolean_array;
use datafusion_common::utils::{compare_rows, extract_row_at_idx_to_buf, get_row_at_idx};
use datafusion_common::{
DataFusionError, Result, ScalarValue, arrow_datafusion_err, internal_err,
- not_impl_err,
};
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
use datafusion_expr::utils::{
@@ -33,6 +32,7 @@ use datafusion_expr::utils::{
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Documentation, EmitTo, Expr, ExprFunctionExt,
GroupsAccumulator, ReversedUDAF, Signature, SortExpr, Volatility,
+ udaf_return_field_preserving_arg_metadata,
};
use datafusion_functions_aggregate_common::utils::get_sort_options;
use datafusion_macros::user_doc;
@@ -284,20 +284,12 @@ impl AggregateUDFImpl for FirstValue {
&self.signature
}
- fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
- not_impl_err!("Not called because the return_field_from_args is implemented")
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ Ok(arg_types[0].clone())
}
fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
- // Preserve metadata from the first argument field
- Ok(Arc::new(
- Field::new(
- self.name(),
- arg_fields[0].data_type().clone(),
- true, // always nullable, there may be no rows
- )
- .with_metadata(arg_fields[0].metadata().clone()),
- ))
+ udaf_return_field_preserving_arg_metadata(self, arg_fields)
}
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
@@ -1342,20 +1334,12 @@ impl AggregateUDFImpl for LastValue {
&self.signature
}
- fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
- not_impl_err!("Not called because the return_field_from_args is implemented")
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ Ok(arg_types[0].clone())
}
fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
- // Preserve metadata from the first argument field
- Ok(Arc::new(
- Field::new(
- self.name(),
- arg_fields[0].data_type().clone(),
- true, // always nullable, there may be no rows
- )
- .with_metadata(arg_fields[0].metadata().clone()),
- ))
+ udaf_return_field_preserving_arg_metadata(self, arg_fields)
}
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
diff --git a/datafusion/functions-aggregate/src/min_max.rs b/datafusion/functions-aggregate/src/min_max.rs
index 88fe168b6f..2af3bbfd5d 100644
--- a/datafusion/functions-aggregate/src/min_max.rs
+++ b/datafusion/functions-aggregate/src/min_max.rs
@@ -11,7 +11,6 @@ mod min_max_struct;
use arrow::array::{ArrayRef, ArrowPrimitiveType, downcast_primitive};
use arrow::datatypes::{DataType, FieldRef, Float16Type, Float32Type, Float64Type};
use datafusion_common::float_semantics::{float_gt, float_lt};
-use datafusion_common::metadata::FieldMetadata;
use datafusion_common::stats::Precision;
use datafusion_common::{ColumnStatistics, Result, exec_err, internal_err};
use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;
@@ -25,7 +24,7 @@ use datafusion_common::ScalarValue;
use datafusion_expr::DistinctHandling;
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Documentation, SetMonotonicity, Signature, Volatility,
- function::AccumulatorArgs, udaf_default_return_field,
+ function::AccumulatorArgs, udaf_return_field_preserving_arg_metadata,
};
use datafusion_expr::{GroupsAccumulator, StatisticsArgs};
use datafusion_macros::user_doc;
@@ -208,16 +207,6 @@ impl FromColumnStatistics for Max {
}
}
-/// Returns the field of the result of `min` or `max`. The result has the type of the argument,
-/// so it keeps the metadata of the argument field, such as an extension type.
-fn min_max_return_field(
- func: &impl AggregateUDFImpl,
- arg_fields: &[FieldRef],
-) -> Result<FieldRef> {
- let field = udaf_default_return_field(func, arg_fields)?;
- Ok(FieldMetadata::from(arg_fields[0].as_ref()).add_to_field_ref(field))
-}
-
impl AggregateUDFImpl for Max {
fn name(&self) -> &str {
"max"
@@ -232,7 +221,7 @@ impl AggregateUDFImpl for Max {
}
fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
- min_max_return_field(self, arg_fields)
+ udaf_return_field_preserving_arg_metadata(self, arg_fields)
}
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
@@ -464,7 +453,7 @@ impl AggregateUDFImpl for Min {
}
fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
- min_max_return_field(self, arg_fields)
+ udaf_return_field_preserving_arg_metadata(self, arg_fields)
}
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
diff --git a/datafusion/functions-aggregate/src/nth_value.rs b/datafusion/functions-aggregate/src/nth_value.rs
index 045f2891cf..674cfa1f6d 100644
--- a/datafusion/functions-aggregate/src/nth_value.rs
+++ b/datafusion/functions-aggregate/src/nth_value.rs
@@ -22,7 +22,7 @@ use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
use datafusion_expr::utils::format_state_name;
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Documentation, ExprFunctionExt, ReversedUDAF,
- Signature, SortExpr, Volatility, lit,
+ Signature, SortExpr, Volatility, lit, udaf_return_field_preserving_arg_metadata,
};
use datafusion_functions_aggregate_common::merge_arrays::merge_ordered_arrays;
use datafusion_functions_aggregate_common::utils::ordering_fields;
@@ -114,6 +114,10 @@ impl AggregateUDFImpl for NthValueAgg {
Ok(arg_types[0].clone())
}
+ fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
+ udaf_return_field_preserving_arg_metadata(self, arg_fields)
+ }
+
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
let n = match acc_args.exprs[1]
.downcast_ref::<Literal>()
diff --git a/datafusion/sqllogictest/test_files/metadata.slt b/datafusion/sqllogictest/test_files/metadata.slt
index 045e8b39b2..b86a6ce3fc 100644
--- a/datafusion/sqllogictest/test_files/metadata.slt
+++ b/datafusion/sqllogictest/test_files/metadata.slt
@@ -346,6 +346,20 @@ from table_with_metadata;
----
3 the id field
+# Regression test: nth_value aggregate should preserve metadata
+query IT
+select nth_value(id, 2 order by id asc nulls last), arrow_metadata(nth_value(id, 2 order by id asc nulls last), 'metadata_key')
+from table_with_metadata;
+----
+3 the id field
+
+# Regression test: any_value should preserve metadata
+query T
+select arrow_metadata(any_value(id), 'metadata_key')
+from table_with_metadata;
+----
+the id field
+
# Regression test: max and min should preserve metadata
query ITIT
select