Apache DataFusion SQL Query Engine
fix: duplicate ordering state field names in partial aggregates (#25196)
Christian Del Montea9579bdfd544
diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rsindex 258d74ecb3..d9234dae3e 100644--- a/datafusion/common/src/dfschema.rs+++ b/datafusion/common/src/dfschema.rs@@ -1138,11 +1138,7 @@ impl TryFrom<SchemaRef> for DFSchema {field_qualifiers: vec![None; field_count],functional_dependencies: FunctionalDependencies::empty(),};- // Without checking names, because schema here may have duplicate field names.- // For example, Partial AggregateMode will generate duplicate field names from- // state_fields.- // See <https://github.com/apache/datafusion/issues/17715>- // dfschema.check_names()?;+ dfschema.check_names()?;Ok(dfschema)}}diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex 6457deeda9..43ceee3444 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -7611,7 +7611,7 @@ async fn execute_logical_plan_rejects_duplicate_unqualified_names_in_replace_vie}#[tokio::test]-async fn test_duplicate_state_fields_for_dfschema_construct() -> Result<()> {+async fn test_partial_aggregate_state_fields_have_unique_names() -> Result<()> {let ctx = SessionContext::new();// Simple schema with just the fields we need@@ -7686,15 +7686,14 @@ async fn test_duplicate_state_fields_for_dfschema_construct() -> Result<()> {).expect("Failed to build partial agg");- // Assert that the schema field names match the expected nameslet expected_field_names = vec!["date","ticker","first_value(value)[first_value]",- "timestamp@0",+ "first_value(value)[ordering_0]","first_value(value)[first_value_is_set]","last_value(value)[last_value]",- "timestamp@0",+ "last_value(value)[ordering_0]","last_value(value)[last_value_is_set]",];@@ -7702,12 +7701,7 @@ async fn test_duplicate_state_fields_for_dfschema_construct() -> Result<()> {let actual_field_names: Vec<_> = binding.fields().iter().map(|f| f.name()).collect();assert_eq!(actual_field_names, expected_field_names);- // Ensure that DFSchema::try_from does not fail- let partial_agg_exec_schema = DFSchema::try_from(partial_agg.schema());- assert!(- partial_agg_exec_schema.is_ok(),- "Expected get AggregateExec schema to succeed with duplicate state fields"- );+ DFSchema::try_from(partial_agg.schema())?;Ok(())}diff --git a/datafusion/expr/src/udaf.rs b/datafusion/expr/src/udaf.rsindex de65702ad9..7358568f0a 100644--- a/datafusion/expr/src/udaf.rs+++ b/datafusion/expr/src/udaf.rs@@ -41,8 +41,7 @@ use crate::function::{};use crate::groups_accumulator::GroupsAccumulator;use crate::udf_eq::UdfEq;-use crate::utils::AggregateOrderSensitivity;-use crate::utils::format_state_name;+use crate::utils::{AggregateOrderSensitivity, format_state_name, ordering_state_fields};use crate::{Accumulator, Expr, expr_vec_fmt};use crate::{Documentation, Signature};@@ -614,7 +613,7 @@ pub trait AggregateUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any {Ok(fields.into_iter().map(Arc::new)- .chain(args.ordering_fields.to_vec())+ .chain(ordering_state_fields(args.name, args.ordering_fields)).collect())}@@ -1762,7 +1761,7 @@ pub enum DistinctHandling {#[cfg(test)]mod test {use crate::{AggregateUDF, AggregateUDFImpl};- use arrow::datatypes::{DataType, FieldRef};+ use arrow::datatypes::{DataType, Field, FieldRef};use datafusion_common::Result;use datafusion_expr_common::accumulator::Accumulator;use datafusion_expr_common::signature::{Signature, Volatility};@@ -1771,6 +1770,7 @@ mod test {};use std::cmp::Ordering;use std::hash::{DefaultHasher, Hash, Hasher};+ use std::sync::Arc;#[derive(Debug, Clone, PartialEq, Eq, Hash)]struct AMeanUdf {@@ -1847,6 +1847,44 @@ mod test {}}+ #[derive(Debug, Clone, PartialEq, Eq, Hash)]+ struct DefaultStateFieldsUdf {+ signature: Signature,+ }++ impl DefaultStateFieldsUdf {+ fn new() -> Self {+ Self {+ signature: Signature::uniform(+ 1,+ vec![DataType::Float64],+ Volatility::Immutable,+ ),+ }+ }+ }++ impl AggregateUDFImpl for DefaultStateFieldsUdf {+ fn name(&self) -> &str {+ "default_state_fields"+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, _args: &[DataType]) -> Result<DataType> {+ unimplemented!()+ }++ fn accumulator(+ &self,+ _acc_args: AccumulatorArgs,+ ) -> Result<Box<dyn Accumulator>> {+ unimplemented!()+ }+ }+#[test]fn test_partial_eq() {let a1 = AggregateUDF::from(AMeanUdf::new());@@ -1875,4 +1913,37 @@ mod test {value.hash(hasher);hasher.finish()}++ #[test]+ fn test_default_state_fields_namespaces_ordering_fields() -> Result<()> {+ let udf = DefaultStateFieldsUdf::new();++ let input_fields = vec![Arc::new(Field::new("value", DataType::Float64, true))];++ let ordering_fields = vec![+ Arc::new(Field::new("timestamp@0", DataType::Int64, true)),+ Arc::new(Field::new("timestamp@0", DataType::Int64, true)),+ ];++ let fields = udf.state_fields(StateFieldsArgs {+ name: "my_agg(value)",+ input_fields: &input_fields,+ return_field: Arc::new(Field::new("result", DataType::Float64, true)),+ ordering_fields: &ordering_fields,+ is_distinct: false,+ })?;++ let names: Vec<_> = fields.iter().map(|f| f.name().as_str()).collect();++ assert_eq!(+ names,+ vec![+ "my_agg(value)[value]",+ "my_agg(value)[ordering_0]",+ "my_agg(value)[ordering_1]",+ ]+ );++ Ok(())+ }}diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rsindex 4ca12c5e03..9462fa1f08 100644--- a/datafusion/expr/src/utils.rs+++ b/datafusion/expr/src/utils.rs@@ -28,7 +28,7 @@ use crate::{};use datafusion_expr_common::signature::{Signature, TypeSignature};-use arrow::datatypes::{DataType, Field, Schema};+use arrow::datatypes::{DataType, Field, FieldRef, Schema};use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion,};@@ -1485,6 +1485,23 @@ pub fn format_state_name(name: &str, state_name: &str) -> String {format!("{name}[{state_name}]")}+/// Creates aggregate state fields for ordering expressions with unique names.+///+/// Each field is renamed using the aggregate name and its ordering position.+pub fn ordering_state_fields(+ name: &str,+ ordering_fields: &[FieldRef],+) -> impl Iterator<Item = FieldRef> {+ ordering_fields.iter().enumerate().map(|(idx, field)| {+ Arc::new(+ field+ .as_ref()+ .clone()+ .with_name(format_state_name(name, &format!("ordering_{idx}"))),+ )+ })+}+/// Determine the set of [`Column`]s produced by the subquery.pub fn collect_subquery_cols(exprs: &[Expr],diff --git a/datafusion/ffi/src/udaf/mod.rs b/datafusion/ffi/src/udaf/mod.rsindex bb0f3567f3..a103b4af20 100644--- a/datafusion/ffi/src/udaf/mod.rs+++ b/datafusion/ffi/src/udaf/mod.rs@@ -812,6 +812,9 @@ mod tests {);let a_field = Arc::new(Field::new("a", DataType::Float64, true));++ let expected_ordering_field =+ Arc::new(a_field.as_ref().clone().with_name("a[ordering_0]"));let state_fields = foreign_udaf.state_fields(StateFieldsArgs {name: "a",input_fields: &[Field::new("f", DataType::Float64, true).into()],@@ -821,7 +824,7 @@ mod tests {})?;assert_eq!(state_fields.len(), 3);- assert_eq!(state_fields[1], a_field);+ assert_eq!(state_fields[1], expected_ordering_field);Ok(())}diff --git a/datafusion/functions-aggregate/src/first_last.rs b/datafusion/functions-aggregate/src/first_last.rsindex d11c12c0ec..d9254c0d2a 100644--- a/datafusion/functions-aggregate/src/first_last.rs+++ b/datafusion/functions-aggregate/src/first_last.rs@@ -40,7 +40,9 @@ use datafusion_common::{not_impl_err,};use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};-use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};+use datafusion_expr::utils::{+ AggregateOrderSensitivity, format_state_name, ordering_state_fields,+};use datafusion_expr::{Accumulator, AggregateUDFImpl, Documentation, EmitTo, Expr, ExprFunctionExt,GroupsAccumulator, ReversedUDAF, Signature, SortExpr, Volatility,@@ -332,7 +334,7 @@ impl AggregateUDFImpl for FirstValue {).into(),];- fields.extend(args.ordering_fields.iter().cloned());+ fields.extend(ordering_state_fields(args.name, args.ordering_fields));fields.push(Field::new(format_state_name(args.name, "first_value_is_set"),@@ -1252,7 +1254,7 @@ impl AggregateUDFImpl for LastValue {).into(),];- fields.extend(args.ordering_fields.iter().cloned());+ fields.extend(ordering_state_fields(args.name, args.ordering_fields));fields.push(Field::new(format_state_name(args.name, "last_value_is_set"),diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 01c6799e96..e11024449a 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -413,6 +413,31 @@ Most existing call sites using `Vec` require no changes. Code that relies onthe exact non-generic function signature of `DataFrame::from_columns` may needto be updated to account for the new generic API.+### Aggregate ordering state field names are now namespaced++Ordering expressions exposed as top-level aggregate state fields now use names+derived from the aggregate name and their ordering position.++For example:++```text+first_value(value)[ordering_0]+```++is now used instead of an unqualified ordering field name such as:++```text+timestamp@0+```++This guarantees unique aggregate state field names and allows+`DFSchema::check_names()` to be enforced.++**Who is affected:**++Users or integrations that inspect aggregate state field names directly,+including custom UDAFs and FFI integrations.+### Physical filter pushdown resolves columns by position`datafusion_physical_plan::filter_pushdown::ChildFilterDescription::from_child`