Apache DataFusion SQL Query Engine
fix: preserve output field name when reversing aggregate window expressions (#24887)
Tim Saucer4c031a267e86
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex 4e32ea3116..9b0ca1b276 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -1336,6 +1336,69 @@ async fn window_aggregates_with_filter() -> Result<()> {Ok(())}+// Test issue: https://github.com/apache/datafusion/issues/24884+//+// When the physical optimizer reverses a window expression to avoid an extra+// sort, the reversed expression must keep its output field name. Otherwise the+// window exec's schema changes while the parent projection still references the+// old column name, and planning fails.+//+// Note this only asserts on planning: executing the plan currently hits a+// separate gap (missing `retract_batch` on the reversed sliding frame), tracked+// by https://github.com/apache/datafusion/issues/24885. Once that is fixed, this+// test can be extended to collect results.+#[tokio::test]+async fn window_reversal_preserves_output_field_names() -> Result<()> {+ fn last_value_over(ascending: bool) -> Expr {+ Expr::from(WindowFunction::new(+ datafusion_functions_aggregate::first_last::last_value_udaf(),+ vec![col("v")],+ ))+ .order_by(vec![col("t").sort(ascending, false)])+ .build()+ .unwrap()+ }++ // `t` must be non-nullable for the ordering equivalence that makes the+ // optimizer reverse the second window instead of adding a second sort.+ let schema = Arc::new(Schema::new(vec![+ Field::new("t", DataType::Int64, false),+ Field::new("v", DataType::Int64, true),+ ]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int64Array::from(vec![1, 2])),+ Arc::new(Int64Array::from(vec![None, Some(10)])),+ ],+ )?;++ let ctx = SessionContext::new();+ let df = ctx+ .read_batch(batch)?+ .with_column("asc_win", last_value_over(true))?+ .with_column("desc_win", last_value_over(false))?;++ let logical_schema = df.schema().clone();+ // Planning used to fail here with an internal error from `EnsureRequirements`.+ let physical_plan = df.create_physical_plan().await?;++ let physical_names = physical_plan+ .schema()+ .fields()+ .iter()+ .map(|f| f.name().clone())+ .collect::<Vec<_>>();+ let logical_names = logical_schema+ .fields()+ .iter()+ .map(|f| f.name().clone())+ .collect::<Vec<_>>();+ assert_eq!(physical_names, logical_names);++ Ok(())+}+// Test issue: https://github.com/apache/datafusion/issues/10346#[tokio::test]async fn test_select_over_aggregate_schema() -> Result<()> {diff --git a/datafusion/core/tests/physical_optimizer/window_optimize.rs b/datafusion/core/tests/physical_optimizer/window_optimize.rsindex 796f6b6259..909cb8b323 100644--- a/datafusion/core/tests/physical_optimizer/window_optimize.rs+++ b/datafusion/core/tests/physical_optimizer/window_optimize.rs@@ -18,17 +18,23 @@#[cfg(test)]mod test {use arrow::array::{Int32Array, RecordBatch};- use arrow_schema::{DataType, Field, Schema};+ use arrow_schema::{DataType, Field, Schema, SortOptions};use datafusion_common::Result;+ use datafusion_common::config::ConfigOptions;use datafusion_datasource::memory::MemorySourceConfig;use datafusion_datasource::source::DataSourceExec;use datafusion_execution::TaskContext;- use datafusion_expr::WindowFrame;+ use datafusion_expr::{WindowFrame, WindowFunctionDefinition};use datafusion_functions_aggregate::count::count_udaf;+ use datafusion_functions_aggregate::first_last::last_value_udaf;use datafusion_physical_expr::aggregate::AggregateExprBuilder;use datafusion_physical_expr::expressions::{Column, col};- use datafusion_physical_expr::window::PlainAggregateWindowExpr;- use datafusion_physical_plan::windows::BoundedWindowAggExec;+ use datafusion_physical_expr::window::{PlainAggregateWindowExpr, WindowExpr};+ use datafusion_physical_expr::{LexOrdering, PhysicalExpr, PhysicalSortExpr};+ use datafusion_physical_optimizer::PhysicalOptimizerRule;+ use datafusion_physical_optimizer::ensure_requirements::EnsureRequirements;+ use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};+ use datafusion_physical_plan::windows::{BoundedWindowAggExec, create_window_expr};use datafusion_physical_plan::{ExecutionPlan, InputOrderMode, common};use std::sync::Arc;@@ -64,6 +70,102 @@ mod test {Ok(())}+ /// Test case for <https://github.com/apache/datafusion/issues/24884>+ ///+ /// `EnsureRequirements` reverses the second window expression to reuse the+ /// `t ASC` ordering required by the first one. That reversal must not rename+ /// the window's output field, otherwise the parent projection -- which+ /// references those columns by name -- can no longer be resolved.+ #[tokio::test]+ async fn test_window_reversal_preserves_output_field_names() -> Result<()> {+ // `t` must be non-nullable for the ordering equivalence that triggers+ // the reversal.+ let schema = Arc::new(Schema::new(vec![+ Field::new("t", DataType::Int32, false),+ Field::new("v", DataType::Int32, true),+ ]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(vec![1, 2])),+ Arc::new(Int32Array::from(vec![None, Some(10)])),+ ],+ )?;+ let ordering =+ LexOrdering::new([PhysicalSortExpr::new_default(col("t", &schema)?)])+ .unwrap();+ let source: Arc<dyn ExecutionPlan> = Arc::new(DataSourceExec::new(Arc::new(+ MemorySourceConfig::try_new(&[vec![batch]], Arc::clone(&schema), None)?+ .try_with_sort_information(vec![ordering])?,+ )));++ // Two `last_value` aggregate-UDAF windows with opposite ORDER BY+ // directions, so the optimizer reverses the second one.+ let window_expr = |ascending: bool| -> Result<Arc<dyn WindowExpr>> {+ let sort_expr = PhysicalSortExpr::new(+ col("t", &schema)?,+ SortOptions::new(!ascending, false),+ );+ create_window_expr(+ &WindowFunctionDefinition::AggregateUDF(last_value_udaf()),+ format!("last_value_{}", if ascending { "asc" } else { "desc" }),+ &[col("v", &schema)?],+ &[],+ std::slice::from_ref(&sort_expr),+ Arc::new(WindowFrame::new(Some(false))),+ Arc::clone(&schema),+ false,+ false,+ None,+ )+ };++ let mut plan: Arc<dyn ExecutionPlan> = source;+ for ascending in [true, false] {+ plan = Arc::new(BoundedWindowAggExec::try_new(+ vec![window_expr(ascending)?],+ plan,+ InputOrderMode::Sorted,+ false,+ )?);+ }++ // A projection that references the window outputs by name, as the+ // physical planner would produce.+ let plan_schema = plan.schema();+ let projection_exprs = plan_schema+ .fields()+ .iter()+ .enumerate()+ .map(|(idx, field)| ProjectionExpr {+ expr: Arc::new(Column::new(field.name(), idx)) as Arc<dyn PhysicalExpr>,+ alias: field.name().clone(),+ })+ .collect::<Vec<_>>();+ let plan: Arc<dyn ExecutionPlan> =+ Arc::new(ProjectionExec::try_new(projection_exprs, plan)?);++ // Used to fail with an internal error from `ProjectionMapping::try_new`.+ let optimized = EnsureRequirements::new()+ .optimize(Arc::clone(&plan), &ConfigOptions::new())?;++ let names = |plan: &Arc<dyn ExecutionPlan>| {+ plan.schema()+ .fields()+ .iter()+ .map(|f| f.name().clone())+ .collect::<Vec<_>>()+ };+ assert_eq!(names(&optimized), names(&plan));++ // The window exec below the projection must keep its field names too,+ // otherwise the projection's columns would dangle.+ let window = optimized.children()[0];+ assert_eq!(names(window), names(&plan));++ Ok(())+ }+pub fn mock_data() -> Result<Arc<DataSourceExec>> {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true),diff --git a/datafusion/physical-expr/src/aggregate.rs b/datafusion/physical-expr/src/aggregate.rsindex 5f045ca8c7..df22bc69d8 100644--- a/datafusion/physical-expr/src/aggregate.rs+++ b/datafusion/physical-expr/src/aggregate.rs@@ -933,24 +933,44 @@ impl AggregateFunctionExpr {/// For aggregates that do not support calculation in reverse,/// returns None (which is the default value).pub fn reverse_expr(&self) -> Option<AggregateFunctionExpr> {+ self.reverse_expr_inner(false)+ }++ /// Same as [`Self::reverse_expr`], but the output `name` is always carried+ /// over unchanged.+ ///+ /// Window execs derive their output schema from `WindowExpr::field()`, which+ /// for aggregate-backed window expressions is this expression's `name`. If+ /// reversal renamed it, the rebuilt window exec would expose a differently+ /// named column while parent plan nodes still reference the old one. This+ /// mirrors `WindowUDFExpr::reverse_expr`, which preserves its name for the+ /// same reason. `AggregateExec`, by contrast, pins its schema at+ /// construction, so it can use [`Self::reverse_expr`] and let the name+ /// reflect the function actually being evaluated.+ pub(crate) fn reverse_expr_preserving_name(&self) -> Option<AggregateFunctionExpr> {+ self.reverse_expr_inner(true)+ }++ fn reverse_expr_inner(&self, preserve_name: bool) -> Option<AggregateFunctionExpr> {match self.fun.reverse_udf() {ReversedUDAF::NotSupported => None,ReversedUDAF::Identical => Some(self.clone()),ReversedUDAF::Reversed(reverse_udf) => {- let was_aliased = self.human_display_alias().is_some();+ let keep_name = preserve_name || self.human_display_alias().is_some();let mut name = self.name().to_string();let mut human_display = self.human_display.clone();// Reversing display follows two paths:- // - aliased display keeps the output `name` unchanged and rewrites only- // the lowered expression in `human_display`.+ // - aliased display (or an explicit request to keep the name) keeps+ // the output `name` unchanged and rewrites only the lowered+ // expression in `human_display`.// - non-aliased display rewrites the canonical `name`, and rewrites// `human_display` only when present.// If the function is changed, we need to reverse order_by clause as well// i.e. First(a order by b asc null first) -> Last(a order by b desc null last)- if !was_aliased && self.fun().name() != reverse_udf.name() {+ if !keep_name && self.fun().name() != reverse_udf.name() {replace_order_by_clause(&mut name);}- if !was_aliased {+ if !keep_name {replace_fn_name_clause(&mut name,self.fun.name(),diff --git a/datafusion/physical-expr/src/window/aggregate.rs b/datafusion/physical-expr/src/window/aggregate.rsindex 7cfdcb167f..a021588edd 100644--- a/datafusion/physical-expr/src/window/aggregate.rs+++ b/datafusion/physical-expr/src/window/aggregate.rs@@ -39,6 +39,53 @@ use datafusion_common::{Result, ScalarValue, exec_datafusion_err};use datafusion_expr::{Accumulator, WindowFrame, WindowFrameBound, WindowFrameUnits};use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;+/// Reverses an aggregate-backed window expression.+///+/// Shared by [`PlainAggregateWindowExpr::get_reverse_expr`] and+/// [`SlidingAggregateWindowExpr::get_reverse_expr`], which are otherwise+/// identical. The reversed frame decides which of the two variants is produced:+/// an ever-expanding frame can be evaluated by the plain (accumulating) variant,+/// anything else needs the sliding variant.+///+/// The aggregate is reversed with+/// [`AggregateFunctionExpr::reverse_expr_preserving_name`] so that the window's+/// output field name is unchanged -- the window exec's schema is derived from+/// [`WindowExpr::field`], and parent plan nodes reference those columns by name.+pub(crate) fn reverse_aggregate_window_expr(+ aggregate: &AggregateFunctionExpr,+ partition_by: &[Arc<dyn PhysicalExpr>],+ order_by: &[PhysicalSortExpr],+ window_frame: &WindowFrame,+ filter: Option<&Arc<dyn PhysicalExpr>>,+) -> Option<Arc<dyn WindowExpr>> {+ aggregate+ .reverse_expr_preserving_name()+ .map(|reverse_expr| {+ let reverse_expr = Arc::new(reverse_expr);+ let reverse_order_by =+ order_by.iter().map(|e| e.reverse()).collect::<Vec<_>>();+ let reverse_window_frame = Arc::new(window_frame.reverse());+ let filter = filter.cloned();+ if reverse_window_frame.is_ever_expanding() {+ Arc::new(PlainAggregateWindowExpr::new(+ reverse_expr,+ partition_by,+ &reverse_order_by,+ reverse_window_frame,+ filter,+ )) as _+ } else {+ Arc::new(SlidingAggregateWindowExpr::new(+ reverse_expr,+ partition_by,+ &reverse_order_by,+ reverse_window_frame,+ filter,+ )) as _+ }+ })+}+/// A window expr that takes the form of an aggregate function.////// See comments on [`WindowExpr`] for more details.@@ -185,34 +232,13 @@ impl WindowExpr for PlainAggregateWindowExpr {}fn get_reverse_expr(&self) -> Option<Arc<dyn WindowExpr>> {- self.aggregate.reverse_expr().map(|reverse_expr| {- let reverse_window_frame = self.window_frame.reverse();- if reverse_window_frame.is_ever_expanding() {- Arc::new(PlainAggregateWindowExpr::new(- Arc::new(reverse_expr),- &self.partition_by.clone(),- &self- .order_by- .iter()- .map(|e| e.reverse())- .collect::<Vec<_>>(),- Arc::new(self.window_frame.reverse()),- self.filter.clone(),- )) as _- } else {- Arc::new(SlidingAggregateWindowExpr::new(- Arc::new(reverse_expr),- &self.partition_by.clone(),- &self- .order_by- .iter()- .map(|e| e.reverse())- .collect::<Vec<_>>(),- Arc::new(self.window_frame.reverse()),- self.filter.clone(),- )) as _- }- })+ reverse_aggregate_window_expr(+ &self.aggregate,+ &self.partition_by,+ &self.order_by,+ &self.window_frame,+ self.filter.as_ref(),+ )}fn uses_bounded_memory(&self) -> bool {diff --git a/datafusion/physical-expr/src/window/sliding_aggregate.rs b/datafusion/physical-expr/src/window/sliding_aggregate.rsindex a39334f057..0954037acb 100644--- a/datafusion/physical-expr/src/window/sliding_aggregate.rs+++ b/datafusion/physical-expr/src/window/sliding_aggregate.rs@@ -22,12 +22,11 @@ use std::ops::Range;use std::sync::Arc;use crate::aggregate::AggregateFunctionExpr;+use crate::window::aggregate::reverse_aggregate_window_expr;use crate::window::window_expr::{AggregateWindowExpr, WindowEvalContext, WindowFn, filter_array,};-use crate::window::{- PartitionBatches, PartitionWindowAggStates, PlainAggregateWindowExpr, WindowExpr,-};+use crate::window::{PartitionBatches, PartitionWindowAggStates, WindowExpr};use crate::{PhysicalExpr, expressions::PhysicalSortExpr};use arrow::array::{ArrayRef, BooleanArray};@@ -122,34 +121,13 @@ impl WindowExpr for SlidingAggregateWindowExpr {}fn get_reverse_expr(&self) -> Option<Arc<dyn WindowExpr>> {- self.aggregate.reverse_expr().map(|reverse_expr| {- let reverse_window_frame = self.window_frame.reverse();- if reverse_window_frame.is_ever_expanding() {- Arc::new(PlainAggregateWindowExpr::new(- Arc::new(reverse_expr),- &self.partition_by.clone(),- &self- .order_by- .iter()- .map(|e| e.reverse())- .collect::<Vec<_>>(),- Arc::new(self.window_frame.reverse()),- self.filter.clone(),- )) as _- } else {- Arc::new(SlidingAggregateWindowExpr::new(- Arc::new(reverse_expr),- &self.partition_by.clone(),- &self- .order_by- .iter()- .map(|e| e.reverse())- .collect::<Vec<_>>(),- Arc::new(self.window_frame.reverse()),- self.filter.clone(),- )) as _- }- })+ reverse_aggregate_window_expr(+ &self.aggregate,+ &self.partition_by,+ &self.order_by,+ &self.window_frame,+ self.filter.as_ref(),+ )}fn uses_bounded_memory(&self) -> bool {diff --git a/datafusion/physical-plan/src/windows/mod.rs b/datafusion/physical-plan/src/windows/mod.rsindex 3f33dfedfd..7c5f55f661 100644--- a/datafusion/physical-plan/src/windows/mod.rs+++ b/datafusion/physical-plan/src/windows/mod.rs@@ -33,7 +33,7 @@ use crate::{use arrow::datatypes::{Schema, SchemaRef};use arrow_schema::{FieldRef, SortOptions};-use datafusion_common::{Result, exec_err};+use datafusion_common::{Result, assert_or_internal_err, exec_err};use datafusion_expr::{LimitEffect, PartitionEvaluator, ReversedUDWF, SetMonotonicity, WindowFrame,WindowFunctionDefinition, WindowUDF,@@ -623,6 +623,20 @@ pub fn get_best_fitting_window(.map(|e| e.get_reverse_expr()).collect::<Option<Vec<_>>>(){+ // The rebuilt exec derives its schema from `WindowExpr::field()`, so a+ // reversal that renames the output field would silently change this+ // node's schema while parent nodes still reference the old column+ // names. Catch that here, where the culprit is identifiable.+ for (reversed, original) in reversed_window_expr.iter().zip(window_exprs) {+ let (reversed_field, original_field) =+ (reversed.field()?, original.field()?);+ assert_or_internal_err!(+ reversed_field.name() == original_field.name(),+ "Reversing window expression changed its output field name from {} to {}",+ original_field.name(),+ reversed_field.name()+ );+ }reversed_window_expr} else {// Cannot take reverse of any of the window expr