Apache DataFusion SQL Query Engine
feat: implement partition_statistics for WindowAggExec (#18534)
Dongpo Liu79f67b8ef2d3
diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rsindex 468d25e0e5..3f079b88d5 100644--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs@@ -30,6 +30,7 @@ mod test {use datafusion_common::{ColumnStatistics, ScalarValue, Statistics};use datafusion_execution::TaskContext;use datafusion_execution::config::SessionConfig;+ use datafusion_expr::{WindowFrame, WindowFunctionDefinition};use datafusion_expr_common::operator::Operator;use datafusion_functions_aggregate::count::count_udaf;use datafusion_physical_expr::Partitioning;@@ -52,6 +53,7 @@ mod test {use datafusion_physical_plan::repartition::RepartitionExec;use datafusion_physical_plan::sorts::sort::SortExec;use datafusion_physical_plan::union::{InterleaveExec, UnionExec};+ use datafusion_physical_plan::windows::{WindowAggExec, create_window_expr};use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties, execute_stream_partitioned,get_plan_string,@@ -1154,4 +1156,105 @@ mod test {Ok(())}++ #[tokio::test]+ async fn test_statistic_by_partition_of_window_agg() -> Result<()> {+ let scan = create_scan_exec_with_statistics(None, Some(2)).await;++ let window_expr = create_window_expr(+ &WindowFunctionDefinition::AggregateUDF(count_udaf()),+ "count".to_owned(),+ &[col("id", &scan.schema())?],+ &[], // no partition by+ &[PhysicalSortExpr::new(+ col("id", &scan.schema())?,+ SortOptions::default(),+ )],+ Arc::new(WindowFrame::new(Some(false))),+ scan.schema(),+ false,+ false,+ None,+ )?;++ let window_agg: Arc<dyn ExecutionPlan> =+ Arc::new(WindowAggExec::try_new(vec![window_expr], scan, true)?);++ // Verify partition statistics are properly propagated (not unknown)+ let statistics = (0..window_agg.output_partitioning().partition_count())+ .map(|idx| window_agg.partition_statistics(Some(idx)))+ .collect::<Result<Vec<_>>>()?;++ assert_eq!(statistics.len(), 2);++ // Window functions preserve input row counts and column statistics+ // but add unknown statistics for the new window column+ let expected_statistic_partition_1 = Statistics {+ num_rows: Precision::Exact(2),+ total_byte_size: Precision::Absent,+ column_statistics: vec![+ ColumnStatistics {+ null_count: Precision::Exact(0),+ max_value: Precision::Exact(ScalarValue::Int32(Some(4))),+ min_value: Precision::Exact(ScalarValue::Int32(Some(3))),+ sum_value: Precision::Absent,+ distinct_count: Precision::Absent,+ byte_size: Precision::Exact(8),+ },+ ColumnStatistics {+ null_count: Precision::Exact(0),+ max_value: Precision::Exact(ScalarValue::Date32(Some(+ DATE_2025_03_02,+ ))),+ min_value: Precision::Exact(ScalarValue::Date32(Some(+ DATE_2025_03_01,+ ))),+ sum_value: Precision::Absent,+ distinct_count: Precision::Absent,+ byte_size: Precision::Exact(8),+ },+ ColumnStatistics::new_unknown(), // window column+ ],+ };++ let expected_statistic_partition_2 = Statistics {+ num_rows: Precision::Exact(2),+ total_byte_size: Precision::Absent,+ column_statistics: vec![+ ColumnStatistics {+ null_count: Precision::Exact(0),+ max_value: Precision::Exact(ScalarValue::Int32(Some(2))),+ min_value: Precision::Exact(ScalarValue::Int32(Some(1))),+ sum_value: Precision::Absent,+ distinct_count: Precision::Absent,+ byte_size: Precision::Exact(8),+ },+ ColumnStatistics {+ null_count: Precision::Exact(0),+ max_value: Precision::Exact(ScalarValue::Date32(Some(+ DATE_2025_03_04,+ ))),+ min_value: Precision::Exact(ScalarValue::Date32(Some(+ DATE_2025_03_03,+ ))),+ sum_value: Precision::Absent,+ distinct_count: Precision::Absent,+ byte_size: Precision::Exact(8),+ },+ ColumnStatistics::new_unknown(), // window column+ ],+ };++ assert_eq!(statistics[0], expected_statistic_partition_1);+ assert_eq!(statistics[1], expected_statistic_partition_2);++ // Verify the statistics match actual execution results+ let expected_stats = vec![+ ExpectedStatistics::NonEmpty(3, 4, 2),+ ExpectedStatistics::NonEmpty(1, 2, 2),+ ];+ validate_statistics_with_data(window_agg, expected_stats, 0).await?;++ Ok(())+ }}diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rsindex d6d5f4fdd2..aa99f4f498 100644--- a/datafusion/physical-plan/src/windows/window_agg_exec.rs+++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs@@ -158,24 +158,6 @@ impl WindowAggExec {.unwrap_or_else(Vec::new)}}-- fn statistics_inner(&self) -> Result<Statistics> {- let input_stat = self.input.partition_statistics(None)?;- let win_cols = self.window_expr.len();- let input_cols = self.input.schema().fields().len();- // TODO stats: some windowing function will maintain invariants such as min, max...- let mut column_statistics = Vec::with_capacity(win_cols + input_cols);- // copy stats of the input to the beginning of the schema.- column_statistics.extend(input_stat.column_statistics);- for _ in 0..win_cols {- column_statistics.push(ColumnStatistics::new_unknown())- }- Ok(Statistics {- num_rows: input_stat.num_rows,- column_statistics,- total_byte_size: Precision::Absent,- })- }}impl DisplayAs for WindowAggExec {@@ -291,15 +273,25 @@ impl ExecutionPlan for WindowAggExec {}fn statistics(&self) -> Result<Statistics> {- self.statistics_inner()+ self.partition_statistics(None)}fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {- if partition.is_none() {- self.statistics_inner()- } else {- Ok(Statistics::new_unknown(&self.schema()))+ let input_stat = self.input.partition_statistics(partition)?;+ let win_cols = self.window_expr.len();+ let input_cols = self.input.schema().fields().len();+ // TODO stats: some windowing function will maintain invariants such as min, max...+ let mut column_statistics = Vec::with_capacity(win_cols + input_cols);+ // copy stats of the input to the beginning of the schema.+ column_statistics.extend(input_stat.column_statistics);+ for _ in 0..win_cols {+ column_statistics.push(ColumnStatistics::new_unknown())}+ Ok(Statistics {+ num_rows: input_stat.num_rows,+ column_statistics,+ total_byte_size: Precision::Absent,+ })}}