Skip to content

Apache DataFusion SQL Query Engine

fix: keep field metadata in the result of min and max (#26939)

Yury Syrovetsky941146372a89
diff --git a/datafusion/functions-aggregate/src/min_max.rs b/datafusion/functions-aggregate/src/min_max.rs
index ca3a61233e..88fe168b6f 100644
--- a/datafusion/functions-aggregate/src/min_max.rs
+++ b/datafusion/functions-aggregate/src/min_max.rs
@@ -9,8 +9,9 @@ mod min_max_bytes;
mod min_max_struct;
use arrow::array::{ArrayRef, ArrowPrimitiveType, downcast_primitive};
-use arrow::datatypes::{DataType, Float16Type, Float32Type, Float64Type};
+use arrow::datatypes::{DataType, FieldRef, Float16Type, Float32Type, Float64Type};
use datafusion_common::float_semantics::{float_gt, float_lt};
+use datafusion_common::metadata::FieldMetadata;
use datafusion_common::stats::Precision;
use datafusion_common::{ColumnStatistics, Result, exec_err, internal_err};
use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;
@@ -24,7 +25,7 @@ use datafusion_common::ScalarValue;
use datafusion_expr::DistinctHandling;
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Documentation, SetMonotonicity, Signature, Volatility,
- function::AccumulatorArgs,
+ function::AccumulatorArgs, udaf_default_return_field,
};
use datafusion_expr::{GroupsAccumulator, StatisticsArgs};
use datafusion_macros::user_doc;
@@ -207,6 +208,16 @@ impl FromColumnStatistics for Max {
}
}
+/// Returns the field of the result of `min` or `max`. The result has the type of the argument,
+/// so it keeps the metadata of the argument field, such as an extension type.
+fn min_max_return_field(
+ func: &impl AggregateUDFImpl,
+ arg_fields: &[FieldRef],
+) -> Result<FieldRef> {
+ let field = udaf_default_return_field(func, arg_fields)?;
+ Ok(FieldMetadata::from(arg_fields[0].as_ref()).add_to_field_ref(field))
+}
+
impl AggregateUDFImpl for Max {
fn name(&self) -> &str {
"max"
@@ -220,6 +231,10 @@ impl AggregateUDFImpl for Max {
Ok(arg_types[0].to_owned())
}
+ fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
+ min_max_return_field(self, arg_fields)
+ }
+
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
Ok(Box::new(MaxAccumulator::try_new(
acc_args.return_field.data_type(),
@@ -448,6 +463,10 @@ impl AggregateUDFImpl for Min {
Ok(arg_types[0].to_owned())
}
+ fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
+ min_max_return_field(self, arg_fields)
+ }
+
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
Ok(Box::new(MinAccumulator::try_new(
acc_args.return_field.data_type(),
diff --git a/datafusion/physical-optimizer/src/aggregate_statistics.rs b/datafusion/physical-optimizer/src/aggregate_statistics.rs
index ab6a0e75c5..b9763ffe28 100644
--- a/datafusion/physical-optimizer/src/aggregate_statistics.rs
+++ b/datafusion/physical-optimizer/src/aggregate_statistics.rs
@@ -5,18 +5,20 @@
//! Utilizing exact statistics from sources to avoid scanning data
use datafusion_common::Result;
use datafusion_common::config::ConfigOptions;
+use datafusion_common::metadata::FieldMetadata;
use datafusion_common::scalar::ScalarValue;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
+use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::aggregates::{
AggregateExec, AggregateInputMode, AggregateMode,
};
+use datafusion_physical_plan::expressions::Literal;
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};
use datafusion_physical_plan::statistics::StatisticsArgs;
use datafusion_physical_plan::udaf::{
AggregateFunctionExpr, StatisticsArgs as PlanStatisticsArgs,
};
-use datafusion_physical_plan::{ExecutionPlan, expressions};
use std::sync::Arc;
use crate::PhysicalOptimizerRule;
@@ -67,8 +69,12 @@ impl PhysicalOptimizerRule for AggregateStatistics {
if let Some((optimizable_statistic, name)) =
take_optimizable_value_from_statistics(&statistics_args, expr)
{
+ // Keep the metadata of the aggregate field, such as an extension type
projections.push(ProjectionExpr {
- expr: expressions::lit(optimizable_statistic),
+ expr: Arc::new(Literal::new_with_metadata(
+ optimizable_statistic,
+ Some(FieldMetadata::from(field.as_ref())),
+ )),
alias: name,
});
} else {
diff --git a/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt b/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt
index a3cd8a18a6..10e6a753fd 100644
--- a/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt
+++ b/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt
@@ -85,6 +85,26 @@ FROM (
----
00010203040506070809000102030506 NULL
+# max and min keep the extension type of their argument. The NULL row makes the column
+# nullable, see https://github.com/apache/datafusion/issues/26469 for a non-null column.
+query ?T?T
+SELECT
+ max(uuid_val),
+ arrow_metadata(max(uuid_val), 'ARROW:extension:name'),
+ min(uuid_val),
+ arrow_metadata(min(uuid_val), 'ARROW:extension:name')
+FROM (
+ SELECT CAST(arrow_cast(raw, 'FixedSizeBinary(16)') AS UUID) AS uuid_val
+ FROM (
+ VALUES
+ (X'00010203040506070809000102030506'),
+ (X'00010203040506070809000102030507'),
+ (NULL)
+ ) AS uuids(raw)
+);
+----
+00010203040506070809000102030507 arrow.uuid 00010203040506070809000102030506 arrow.uuid
+
# Extension types of Parquet columns are visible to expressions evaluated in the scan,
# also when the scan replaces a constant column with a literal from the statistics
statement ok
diff --git a/datafusion/sqllogictest/test_files/metadata.slt b/datafusion/sqllogictest/test_files/metadata.slt
index 5e0d5d6737..045e8b39b2 100644
--- a/datafusion/sqllogictest/test_files/metadata.slt
+++ b/datafusion/sqllogictest/test_files/metadata.slt
@@ -346,6 +346,30 @@ from table_with_metadata;
----
3 the id field
+# Regression test: max and min should preserve metadata
+query ITIT
+select
+ max(id), arrow_metadata(max(id), 'metadata_key'),
+ min(id), arrow_metadata(min(id), 'metadata_key')
+from table_with_metadata;
+----
+3 the id field 1 the id field
+
+# Regression test: max and min with GROUP BY (groups accumulator) should preserve metadata
+query ITTT
+select
+ id,
+ arrow_metadata(max(name), 'metadata_key'),
+ arrow_metadata(min(name), 'metadata_key'),
+ max(name)
+from table_with_metadata
+group by id
+order by id asc nulls last;
+----
+1 the name field the name field NULL
+3 the name field the name field baz
+NULL the name field the name field bar
+
# Regression test: DISTINCT ON should preserve metadata (uses first_value internally)
query ITTT
select distinct on (id) id, arrow_metadata(id, 'metadata_key'), name, arrow_metadata(name, 'metadata_key')
@@ -498,6 +522,15 @@ from table_with_metadata limit 1;
----
1 the id field
+# Test MAX and MIN window functions preserve metadata
+query IT
+select
+ max(id) over (order by id asc nulls last) as mx,
+ arrow_metadata(min(id) over (order by id asc nulls last), 'metadata_key') as meta
+from table_with_metadata limit 1;
+----
+1 the id field
+
# Test LAST_VALUE window function preserves metadata
query IT
select
@@ -548,6 +581,41 @@ select arrow_metadata((select id from table_with_metadata where id = 1));
----
{metadata_key: the id field}
+# Regression test: max and min answered from Parquet statistics should preserve metadata
+statement ok
+COPY (SELECT id FROM table_with_metadata) TO 'test_files/scratch/metadata/min_max.parquet';
+
+statement ok
+SET datafusion.execution.parquet.skip_metadata = false;
+
+statement ok
+CREATE EXTERNAL TABLE min_max_with_metadata STORED AS PARQUET
+LOCATION 'test_files/scratch/metadata/min_max.parquet';
+
+query ITIT
+SELECT
+ max(id), arrow_metadata(max(id), 'metadata_key'),
+ min(id), arrow_metadata(min(id), 'metadata_key')
+FROM min_max_with_metadata;
+----
+3 the id field 1 the id field
+
+query TT
+EXPLAIN SELECT max(id), min(id) FROM min_max_with_metadata;
+----
+logical_plan
+01)Aggregate: groupBy=[[]], aggr=[[max(min_max_with_metadata.id), min(min_max_with_metadata.id)]]
+02)--TableScan: min_max_with_metadata projection=[id]
+physical_plan
+01)ProjectionExec: expr=[3 as max(min_max_with_metadata.id), 1 as min(min_max_with_metadata.id)]
+02)--PlaceholderRowExec
+
+statement ok
+DROP TABLE min_max_with_metadata;
+
+statement ok
+RESET datafusion.execution.parquet.skip_metadata;
+
# coalesce, nvl, ifnull and nvl2: conflicting extension types are a planning error
statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'
select coalesce(