Skip to content

Apache DataFusion SQL Query Engine

fix: duplicate ordering state field names in partial aggregates (#25196)

Christian Del Montea9579bdfd544
diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs
index 258d74ecb3..d9234dae3e 100644
--- a/datafusion/common/src/dfschema.rs
+++ b/datafusion/common/src/dfschema.rs
@@ -1138,11 +1138,7 @@ impl TryFrom<SchemaRef> for DFSchema {
field_qualifiers: vec![None; field_count],
functional_dependencies: FunctionalDependencies::empty(),
};
- // Without checking names, because schema here may have duplicate field names.
- // For example, Partial AggregateMode will generate duplicate field names from
- // state_fields.
- // See <https://github.com/apache/datafusion/issues/17715>
- // dfschema.check_names()?;
+ dfschema.check_names()?;
Ok(dfschema)
}
}
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs
index 6457deeda9..43ceee3444 100644
--- a/datafusion/core/tests/dataframe/mod.rs
+++ b/datafusion/core/tests/dataframe/mod.rs
@@ -7611,7 +7611,7 @@ async fn execute_logical_plan_rejects_duplicate_unqualified_names_in_replace_vie
}
#[tokio::test]
-async fn test_duplicate_state_fields_for_dfschema_construct() -> Result<()> {
+async fn test_partial_aggregate_state_fields_have_unique_names() -> Result<()> {
let ctx = SessionContext::new();
// Simple schema with just the fields we need
@@ -7686,15 +7686,14 @@ async fn test_duplicate_state_fields_for_dfschema_construct() -> Result<()> {
)
.expect("Failed to build partial agg");
- // Assert that the schema field names match the expected names
let expected_field_names = vec![
"date",
"ticker",
"first_value(value)[first_value]",
- "timestamp@0",
+ "first_value(value)[ordering_0]",
"first_value(value)[first_value_is_set]",
"last_value(value)[last_value]",
- "timestamp@0",
+ "last_value(value)[ordering_0]",
"last_value(value)[last_value_is_set]",
];
@@ -7702,12 +7701,7 @@ async fn test_duplicate_state_fields_for_dfschema_construct() -> Result<()> {
let actual_field_names: Vec<_> = binding.fields().iter().map(|f| f.name()).collect();
assert_eq!(actual_field_names, expected_field_names);
- // Ensure that DFSchema::try_from does not fail
- let partial_agg_exec_schema = DFSchema::try_from(partial_agg.schema());
- assert!(
- partial_agg_exec_schema.is_ok(),
- "Expected get AggregateExec schema to succeed with duplicate state fields"
- );
+ DFSchema::try_from(partial_agg.schema())?;
Ok(())
}
diff --git a/datafusion/expr/src/udaf.rs b/datafusion/expr/src/udaf.rs
index de65702ad9..7358568f0a 100644
--- a/datafusion/expr/src/udaf.rs
+++ b/datafusion/expr/src/udaf.rs
@@ -41,8 +41,7 @@ use crate::function::{
};
use crate::groups_accumulator::GroupsAccumulator;
use crate::udf_eq::UdfEq;
-use crate::utils::AggregateOrderSensitivity;
-use crate::utils::format_state_name;
+use crate::utils::{AggregateOrderSensitivity, format_state_name, ordering_state_fields};
use crate::{Accumulator, Expr, expr_vec_fmt};
use crate::{Documentation, Signature};
@@ -614,7 +613,7 @@ pub trait AggregateUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any {
Ok(fields
.into_iter()
.map(Arc::new)
- .chain(args.ordering_fields.to_vec())
+ .chain(ordering_state_fields(args.name, args.ordering_fields))
.collect())
}
@@ -1762,7 +1761,7 @@ pub enum DistinctHandling {
#[cfg(test)]
mod test {
use crate::{AggregateUDF, AggregateUDFImpl};
- use arrow::datatypes::{DataType, FieldRef};
+ use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::Result;
use datafusion_expr_common::accumulator::Accumulator;
use datafusion_expr_common::signature::{Signature, Volatility};
@@ -1771,6 +1770,7 @@ mod test {
};
use std::cmp::Ordering;
use std::hash::{DefaultHasher, Hash, Hasher};
+ use std::sync::Arc;
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct AMeanUdf {
@@ -1847,6 +1847,44 @@ mod test {
}
}
+ #[derive(Debug, Clone, PartialEq, Eq, Hash)]
+ struct DefaultStateFieldsUdf {
+ signature: Signature,
+ }
+
+ impl DefaultStateFieldsUdf {
+ fn new() -> Self {
+ Self {
+ signature: Signature::uniform(
+ 1,
+ vec![DataType::Float64],
+ Volatility::Immutable,
+ ),
+ }
+ }
+ }
+
+ impl AggregateUDFImpl for DefaultStateFieldsUdf {
+ fn name(&self) -> &str {
+ "default_state_fields"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _args: &[DataType]) -> Result<DataType> {
+ unimplemented!()
+ }
+
+ fn accumulator(
+ &self,
+ _acc_args: AccumulatorArgs,
+ ) -> Result<Box<dyn Accumulator>> {
+ unimplemented!()
+ }
+ }
+
#[test]
fn test_partial_eq() {
let a1 = AggregateUDF::from(AMeanUdf::new());
@@ -1875,4 +1913,37 @@ mod test {
value.hash(hasher);
hasher.finish()
}
+
+ #[test]
+ fn test_default_state_fields_namespaces_ordering_fields() -> Result<()> {
+ let udf = DefaultStateFieldsUdf::new();
+
+ let input_fields = vec![Arc::new(Field::new("value", DataType::Float64, true))];
+
+ let ordering_fields = vec![
+ Arc::new(Field::new("timestamp@0", DataType::Int64, true)),
+ Arc::new(Field::new("timestamp@0", DataType::Int64, true)),
+ ];
+
+ let fields = udf.state_fields(StateFieldsArgs {
+ name: "my_agg(value)",
+ input_fields: &input_fields,
+ return_field: Arc::new(Field::new("result", DataType::Float64, true)),
+ ordering_fields: &ordering_fields,
+ is_distinct: false,
+ })?;
+
+ let names: Vec<_> = fields.iter().map(|f| f.name().as_str()).collect();
+
+ assert_eq!(
+ names,
+ vec![
+ "my_agg(value)[value]",
+ "my_agg(value)[ordering_0]",
+ "my_agg(value)[ordering_1]",
+ ]
+ );
+
+ Ok(())
+ }
}
diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rs
index 4ca12c5e03..9462fa1f08 100644
--- a/datafusion/expr/src/utils.rs
+++ b/datafusion/expr/src/utils.rs
@@ -28,7 +28,7 @@ use crate::{
};
use datafusion_expr_common::signature::{Signature, TypeSignature};
-use arrow::datatypes::{DataType, Field, Schema};
+use arrow::datatypes::{DataType, Field, FieldRef, Schema};
use datafusion_common::tree_node::{
Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
};
@@ -1485,6 +1485,23 @@ pub fn format_state_name(name: &str, state_name: &str) -> String {
format!("{name}[{state_name}]")
}
+/// Creates aggregate state fields for ordering expressions with unique names.
+///
+/// Each field is renamed using the aggregate name and its ordering position.
+pub fn ordering_state_fields(
+ name: &str,
+ ordering_fields: &[FieldRef],
+) -> impl Iterator<Item = FieldRef> {
+ ordering_fields.iter().enumerate().map(|(idx, field)| {
+ Arc::new(
+ field
+ .as_ref()
+ .clone()
+ .with_name(format_state_name(name, &format!("ordering_{idx}"))),
+ )
+ })
+}
+
/// Determine the set of [`Column`]s produced by the subquery.
pub fn collect_subquery_cols(
exprs: &[Expr],
diff --git a/datafusion/ffi/src/udaf/mod.rs b/datafusion/ffi/src/udaf/mod.rs
index bb0f3567f3..a103b4af20 100644
--- a/datafusion/ffi/src/udaf/mod.rs
+++ b/datafusion/ffi/src/udaf/mod.rs
@@ -812,6 +812,9 @@ mod tests {
);
let a_field = Arc::new(Field::new("a", DataType::Float64, true));
+
+ let expected_ordering_field =
+ Arc::new(a_field.as_ref().clone().with_name("a[ordering_0]"));
let state_fields = foreign_udaf.state_fields(StateFieldsArgs {
name: "a",
input_fields: &[Field::new("f", DataType::Float64, true).into()],
@@ -821,7 +824,7 @@ mod tests {
})?;
assert_eq!(state_fields.len(), 3);
- assert_eq!(state_fields[1], a_field);
+ assert_eq!(state_fields[1], expected_ordering_field);
Ok(())
}
diff --git a/datafusion/functions-aggregate/src/first_last.rs b/datafusion/functions-aggregate/src/first_last.rs
index d11c12c0ec..d9254c0d2a 100644
--- a/datafusion/functions-aggregate/src/first_last.rs
+++ b/datafusion/functions-aggregate/src/first_last.rs
@@ -40,7 +40,9 @@ use datafusion_common::{
not_impl_err,
};
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
-use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
+use datafusion_expr::utils::{
+ AggregateOrderSensitivity, format_state_name, ordering_state_fields,
+};
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Documentation, EmitTo, Expr, ExprFunctionExt,
GroupsAccumulator, ReversedUDAF, Signature, SortExpr, Volatility,
@@ -332,7 +334,7 @@ impl AggregateUDFImpl for FirstValue {
)
.into(),
];
- fields.extend(args.ordering_fields.iter().cloned());
+ fields.extend(ordering_state_fields(args.name, args.ordering_fields));
fields.push(
Field::new(
format_state_name(args.name, "first_value_is_set"),
@@ -1252,7 +1254,7 @@ impl AggregateUDFImpl for LastValue {
)
.into(),
];
- fields.extend(args.ordering_fields.iter().cloned());
+ fields.extend(ordering_state_fields(args.name, args.ordering_fields));
fields.push(
Field::new(
format_state_name(args.name, "last_value_is_set"),
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index 01c6799e96..e11024449a 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -413,6 +413,31 @@ Most existing call sites using `Vec` require no changes. Code that relies on
the exact non-generic function signature of `DataFrame::from_columns` may need
to be updated to account for the new generic API.
+### Aggregate ordering state field names are now namespaced
+
+Ordering expressions exposed as top-level aggregate state fields now use names
+derived from the aggregate name and their ordering position.
+
+For example:
+
+```text
+first_value(value)[ordering_0]
+```
+
+is now used instead of an unqualified ordering field name such as:
+
+```text
+timestamp@0
+```
+
+This guarantees unique aggregate state field names and allows
+`DFSchema::check_names()` to be enforced.
+
+**Who is affected:**
+
+Users or integrations that inspect aggregate state field names directly,
+including custom UDAFs and FFI integrations.
+
### Physical filter pushdown resolves columns by position
`datafusion_physical_plan::filter_pushdown::ChildFilterDescription::from_child`