Apache DataFusion SQL Query Engine
fix: keep field metadata in the result of min and max (#26939)
Yury Syrovetsky941146372a89
diff --git a/datafusion/functions-aggregate/src/min_max.rs b/datafusion/functions-aggregate/src/min_max.rsindex ca3a61233e..88fe168b6f 100644--- a/datafusion/functions-aggregate/src/min_max.rs+++ b/datafusion/functions-aggregate/src/min_max.rs@@ -9,8 +9,9 @@ mod min_max_bytes;mod min_max_struct;use arrow::array::{ArrayRef, ArrowPrimitiveType, downcast_primitive};-use arrow::datatypes::{DataType, Float16Type, Float32Type, Float64Type};+use arrow::datatypes::{DataType, FieldRef, Float16Type, Float32Type, Float64Type};use datafusion_common::float_semantics::{float_gt, float_lt};+use datafusion_common::metadata::FieldMetadata;use datafusion_common::stats::Precision;use datafusion_common::{ColumnStatistics, Result, exec_err, internal_err};use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;@@ -24,7 +25,7 @@ use datafusion_common::ScalarValue;use datafusion_expr::DistinctHandling;use datafusion_expr::{Accumulator, AggregateUDFImpl, Documentation, SetMonotonicity, Signature, Volatility,- function::AccumulatorArgs,+ function::AccumulatorArgs, udaf_default_return_field,};use datafusion_expr::{GroupsAccumulator, StatisticsArgs};use datafusion_macros::user_doc;@@ -207,6 +208,16 @@ impl FromColumnStatistics for Max {}}+/// Returns the field of the result of `min` or `max`. The result has the type of the argument,+/// so it keeps the metadata of the argument field, such as an extension type.+fn min_max_return_field(+ func: &impl AggregateUDFImpl,+ arg_fields: &[FieldRef],+) -> Result<FieldRef> {+ let field = udaf_default_return_field(func, arg_fields)?;+ Ok(FieldMetadata::from(arg_fields[0].as_ref()).add_to_field_ref(field))+}+impl AggregateUDFImpl for Max {fn name(&self) -> &str {"max"@@ -220,6 +231,10 @@ impl AggregateUDFImpl for Max {Ok(arg_types[0].to_owned())}+ fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {+ min_max_return_field(self, arg_fields)+ }+fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {Ok(Box::new(MaxAccumulator::try_new(acc_args.return_field.data_type(),@@ -448,6 +463,10 @@ impl AggregateUDFImpl for Min {Ok(arg_types[0].to_owned())}+ fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {+ min_max_return_field(self, arg_fields)+ }+fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {Ok(Box::new(MinAccumulator::try_new(acc_args.return_field.data_type(),diff --git a/datafusion/physical-optimizer/src/aggregate_statistics.rs b/datafusion/physical-optimizer/src/aggregate_statistics.rsindex ab6a0e75c5..b9763ffe28 100644--- a/datafusion/physical-optimizer/src/aggregate_statistics.rs+++ b/datafusion/physical-optimizer/src/aggregate_statistics.rs@@ -5,18 +5,20 @@//! Utilizing exact statistics from sources to avoid scanning datause datafusion_common::Result;use datafusion_common::config::ConfigOptions;+use datafusion_common::metadata::FieldMetadata;use datafusion_common::scalar::ScalarValue;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};+use datafusion_physical_plan::ExecutionPlan;use datafusion_physical_plan::aggregates::{AggregateExec, AggregateInputMode, AggregateMode,};+use datafusion_physical_plan::expressions::Literal;use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};use datafusion_physical_plan::statistics::StatisticsArgs;use datafusion_physical_plan::udaf::{AggregateFunctionExpr, StatisticsArgs as PlanStatisticsArgs,};-use datafusion_physical_plan::{ExecutionPlan, expressions};use std::sync::Arc;use crate::PhysicalOptimizerRule;@@ -67,8 +69,12 @@ impl PhysicalOptimizerRule for AggregateStatistics {if let Some((optimizable_statistic, name)) =take_optimizable_value_from_statistics(&statistics_args, expr){+ // Keep the metadata of the aggregate field, such as an extension typeprojections.push(ProjectionExpr {- expr: expressions::lit(optimizable_statistic),+ expr: Arc::new(Literal::new_with_metadata(+ optimizable_statistic,+ Some(FieldMetadata::from(field.as_ref())),+ )),alias: name,});} else {diff --git a/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt b/datafusion/sqllogictest/test_files/cast_extension_type_metadata.sltindex a3cd8a18a6..10e6a753fd 100644--- a/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt+++ b/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt@@ -85,6 +85,26 @@ FROM (----00010203040506070809000102030506 NULL+# max and min keep the extension type of their argument. The NULL row makes the column+# nullable, see https://github.com/apache/datafusion/issues/26469 for a non-null column.+query ?T?T+SELECT+ max(uuid_val),+ arrow_metadata(max(uuid_val), 'ARROW:extension:name'),+ min(uuid_val),+ arrow_metadata(min(uuid_val), 'ARROW:extension:name')+FROM (+ SELECT CAST(arrow_cast(raw, 'FixedSizeBinary(16)') AS UUID) AS uuid_val+ FROM (+ VALUES+ (X'00010203040506070809000102030506'),+ (X'00010203040506070809000102030507'),+ (NULL)+ ) AS uuids(raw)+);+----+00010203040506070809000102030507 arrow.uuid 00010203040506070809000102030506 arrow.uuid+# Extension types of Parquet columns are visible to expressions evaluated in the scan,# also when the scan replaces a constant column with a literal from the statisticsstatement okdiff --git a/datafusion/sqllogictest/test_files/metadata.slt b/datafusion/sqllogictest/test_files/metadata.sltindex 5e0d5d6737..045e8b39b2 100644--- a/datafusion/sqllogictest/test_files/metadata.slt+++ b/datafusion/sqllogictest/test_files/metadata.slt@@ -346,6 +346,30 @@ from table_with_metadata;----3 the id field+# Regression test: max and min should preserve metadata+query ITIT+select+ max(id), arrow_metadata(max(id), 'metadata_key'),+ min(id), arrow_metadata(min(id), 'metadata_key')+from table_with_metadata;+----+3 the id field 1 the id field++# Regression test: max and min with GROUP BY (groups accumulator) should preserve metadata+query ITTT+select+ id,+ arrow_metadata(max(name), 'metadata_key'),+ arrow_metadata(min(name), 'metadata_key'),+ max(name)+from table_with_metadata+group by id+order by id asc nulls last;+----+1 the name field the name field NULL+3 the name field the name field baz+NULL the name field the name field bar+# Regression test: DISTINCT ON should preserve metadata (uses first_value internally)query ITTTselect distinct on (id) id, arrow_metadata(id, 'metadata_key'), name, arrow_metadata(name, 'metadata_key')@@ -498,6 +522,15 @@ from table_with_metadata limit 1;----1 the id field+# Test MAX and MIN window functions preserve metadata+query IT+select+ max(id) over (order by id asc nulls last) as mx,+ arrow_metadata(min(id) over (order by id asc nulls last), 'metadata_key') as meta+from table_with_metadata limit 1;+----+1 the id field+# Test LAST_VALUE window function preserves metadataquery ITselect@@ -548,6 +581,41 @@ select arrow_metadata((select id from table_with_metadata where id = 1));----{metadata_key: the id field}+# Regression test: max and min answered from Parquet statistics should preserve metadata+statement ok+COPY (SELECT id FROM table_with_metadata) TO 'test_files/scratch/metadata/min_max.parquet';++statement ok+SET datafusion.execution.parquet.skip_metadata = false;++statement ok+CREATE EXTERNAL TABLE min_max_with_metadata STORED AS PARQUET+LOCATION 'test_files/scratch/metadata/min_max.parquet';++query ITIT+SELECT+ max(id), arrow_metadata(max(id), 'metadata_key'),+ min(id), arrow_metadata(min(id), 'metadata_key')+FROM min_max_with_metadata;+----+3 the id field 1 the id field++query TT+EXPLAIN SELECT max(id), min(id) FROM min_max_with_metadata;+----+logical_plan+01)Aggregate: groupBy=[[]], aggr=[[max(min_max_with_metadata.id), min(min_max_with_metadata.id)]]+02)--TableScan: min_max_with_metadata projection=[id]+physical_plan+01)ProjectionExec: expr=[3 as max(min_max_with_metadata.id), 1 as min(min_max_with_metadata.id)]+02)--PlaceholderRowExec++statement ok+DROP TABLE min_max_with_metadata;++statement ok+RESET datafusion.execution.parquet.skip_metadata;+# coalesce, nvl, ifnull and nvl2: conflicting extension types are a planning errorstatement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'select coalesce(