Apache DataFusion SQL Query Engine
fix: ignore constant aggregate completion boundaries (#25019)
Yash Bhisikar6ff484a96d7e
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex 5db3d23ca1..6457deeda9 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -910,7 +910,7 @@ async fn test_aggregate_with_pk2() -> Result<()> {assert_snapshot!(physical_plan_to_string(&df).await,@r"- AggregateExec: mode=Single, gby=[id@0 as id, name@1 as name], aggr=[], ordering_mode=Sorted+ AggregateExec: mode=Single, gby=[id@0 as id, name@1 as name], aggr=[]FilterExec: id@0 = 1 AND name@1 = aDataSourceExec: partitions=1, partition_sizes=[1]"@@ -958,7 +958,7 @@ async fn test_aggregate_with_pk3() -> Result<()> {assert_snapshot!(physical_plan_to_string(&df).await,@r"- AggregateExec: mode=Single, gby=[id@0 as id, name@1 as name], aggr=[], ordering_mode=PartiallySorted([0])+ AggregateExec: mode=Single, gby=[id@0 as id, name@1 as name], aggr=[]FilterExec: id@0 = 1DataSourceExec: partitions=1, partition_sizes=[1]"@@ -1008,7 +1008,7 @@ async fn test_aggregate_with_pk4() -> Result<()> {assert_snapshot!(physical_plan_to_string(&df).await,@r"- AggregateExec: mode=Single, gby=[id@0 as id], aggr=[], ordering_mode=Sorted+ AggregateExec: mode=Single, gby=[id@0 as id], aggr=[]FilterExec: id@0 = 1DataSourceExec: partitions=1, partition_sizes=[1]"diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rsindex 783c216f20..1ad268c4b4 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs@@ -740,12 +740,12 @@ fn test_pushdown_through_aggregates_on_grouping_columns() {OptimizationTest:input:- FilterExec: b@1 = bar- - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt], ordering_mode=PartiallySorted([0])+ - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt]- FilterExec: a@0 = foo- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=trueoutput:Ok:- - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt], ordering_mode=Sorted+ - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt]- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = foo AND b@1 = bar");@@ -804,7 +804,7 @@ fn test_pushdown_through_aggregates_preserves_parent_filter_order() {output:Ok:- FilterExec: cnt@2 = 1- - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt], ordering_mode=PartiallySorted([1])+ - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt]- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar");@@ -3206,7 +3206,7 @@ fn test_pushdown_filter_on_non_first_grouping_column() {- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=trueoutput:Ok:- - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt], ordering_mode=PartiallySorted([1])+ - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt]- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar");@@ -3542,7 +3542,7 @@ fn test_pushdown_through_aggregate_with_reordered_input_columns() {- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=trueoutput:Ok:- - AggregateExec: mode=Final, gby=[a@1 as a, b@2 as b], aggr=[cnt], ordering_mode=PartiallySorted([1])+ - AggregateExec: mode=Final, gby=[a@1 as a, b@2 as b], aggr=[cnt]- ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar"diff --git a/datafusion/core/tests/physical_optimizer/sanity_checker.rs b/datafusion/core/tests/physical_optimizer/sanity_checker.rsindex 32f02d0f6a..800d459c33 100644--- a/datafusion/core/tests/physical_optimizer/sanity_checker.rs+++ b/datafusion/core/tests/physical_optimizer/sanity_checker.rs@@ -321,6 +321,27 @@ async fn test_aggregate() -> Result<()> {Ok(())}+#[tokio::test]+async fn test_constant_grouping_expr_on_unbounded_input() -> Result<()> {+ let case = QueryCase {+ sql: "SELECT c1, c2, MIN(c4) FROM test WHERE c2 = 1 GROUP BY c1, c2".to_string(),+ cases: vec![+ Arc::new(UnaryTestCase {+ source_type: SourceType::Bounded,+ expect_fail: false,+ }),+ Arc::new(UnaryTestCase {+ source_type: SourceType::Unbounded,+ expect_fail: true,+ }),+ ],+ error_operator: "operator: AggregateExec".to_string(),+ };++ case.run().await?;+ Ok(())+}+#[tokio::test]async fn test_window_agg_hash_partition() -> Result<()> {let test1 = UnaryTestCase {diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 49a726f9be..f2afe6e754 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -1033,6 +1033,8 @@ impl AggregateExec {let required_input_ordering =LexRequirement::new(new_requirements).map(OrderingRequirements::new_soft);+ // Constant expressions never change, so they cannot mark a completed group.+ // Exclude them from both the ordering indices and the group expression count.// If our aggregation has grouping sets then our base grouping exprs will// be expanded based on the flags in `group_by.groups` where for each// group we swap the grouping expr for `null` if the flag is `true`@@ -1041,9 +1043,18 @@ impl AggregateExec {let indices: Vec<usize> = indices.into_iter().filter(|idx| group_by.groups.iter().all(|group| !group[*idx]))+ .filter(|idx| {+ input_eq_properties+ .is_expr_constant(&groupby_exprs[*idx])+ .is_none()+ }).collect();- let mut input_order_mode = if indices.len() == groupby_exprs.len()+ let num_non_constant_groupby_exprs = groupby_exprs+ .iter()+ .filter(|expr| input_eq_properties.is_expr_constant(expr).is_none())+ .count();+ let mut input_order_mode = if indices.len() == num_non_constant_groupby_exprs&& !indices.is_empty()&& group_by.groups.len() == 1{@@ -3228,7 +3239,7 @@ mod tests {use arrow::array::{BooleanArray, DictionaryArray, Float32Array, Float64Array, Int32Array,- Int64Array, NullArray, StructArray, UInt32Array, UInt64Array,+ Int64Array, NullArray, StringArray, StructArray, UInt32Array, UInt64Array,};use arrow::compute::{SortOptions, concat_batches};use arrow::datatypes::Int32Type;@@ -3239,7 +3250,7 @@ mod tests {use datafusion_execution::runtime_env::RuntimeEnvBuilder;use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};use datafusion_expr::{- Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator,+ Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator, Operator,Signature, Volatility,};use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;@@ -3253,7 +3264,7 @@ mod tests {use datafusion_physical_expr::Partitioning;use datafusion_physical_expr::PhysicalSortExpr;use datafusion_physical_expr::aggregate::AggregateExprBuilder;- use datafusion_physical_expr::expressions::{Literal, NotExpr};+ use datafusion_physical_expr::expressions::{Literal, NotExpr, binary};use crate::projection::ProjectionExec;use crate::repartition::RepartitionExec;@@ -4697,6 +4708,87 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn constant_grouping_expr_is_not_a_completion_boundary() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("key", DataType::Int32, true),+ Field::new("market", DataType::Utf8, true),+ Field::new("value", DataType::Int64, false),+ ]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(vec![None, Some(10), Some(10)])),+ Arc::new(StringArray::from(vec![Some("US"), Some("US"), Some("US")])),+ Arc::new(Int64Array::from(vec![3, 1, 2])),+ ],+ )?;++ let build_aggregate = |input: Arc<dyn ExecutionPlan>| -> Result<AggregateExec> {+ let predicate =+ binary(col("market", &schema)?, Operator::Eq, lit("US"), &schema)?;+ let input = Arc::new(FilterExecBuilder::new(predicate, input).build()?);+ AggregateExec::try_new(+ AggregateMode::Single,+ PhysicalGroupBy::new_single(vec![+ (col("key", &schema)?, "key".to_string()),+ (col("market", &schema)?, "market".to_string()),+ ]),+ vec![Arc::new(+ AggregateExprBuilder::new(count_udaf(), vec![col("value", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("COUNT(value)")+ .build()?,+ )],+ vec![None],+ input,+ Arc::clone(&schema),+ )+ };++ let unordered_input = TestMemoryExec::try_new_exec(+ &[vec![batch.clone()]],+ Arc::clone(&schema),+ None,+ )?;+ let aggregate = build_aggregate(unordered_input)?;+ assert_eq!(aggregate.input_order_mode(), &InputOrderMode::Linear);+ assert_eq!(+ aggregate.schema().as_ref(),+ &Schema::new(vec![+ Field::new("key", DataType::Int32, true),+ Field::new("market", DataType::Utf8, true),+ Field::new("COUNT(value)", DataType::Int64, false),+ ])+ );++ let output =+ collect(aggregate.execute(0, Arc::new(TaskContext::default()))?).await?;+ assert_eq!(output.iter().map(RecordBatch::num_rows).sum::<usize>(), 2);+ assert_snapshot!(batches_to_sort_string(&output), @r"++-----+--------+--------------++| key | market | COUNT(value) |++-----+--------+--------------++| | US | 1 |+| 10 | US | 2 |++-----+--------+--------------++");++ let ordering = LexOrdering::new([PhysicalSortExpr::new_default(Arc::new(+ Column::new("key", 0),+ ))])+ .unwrap();+ let ordered_input =+ TestMemoryExec::try_new(&[vec![batch]], Arc::clone(&schema), None)?+ .try_with_sort_information(vec![ordering])?;+ let ordered_input =+ Arc::new(TestMemoryExec::update_cache(&Arc::new(ordered_input)));+ let aggregate = build_aggregate(ordered_input)?;+ assert_eq!(aggregate.input_order_mode(), &InputOrderMode::Sorted);++ Ok(())+ }+fn partial_reduce_test_aggregate() -> Result<AggregateExec> {partial_reduce_test_aggregate_with_batches(1)}@@ -6584,7 +6676,10 @@ mod tests {");}}- Err(e) => assert!(matches!(e, DataFusionError::ResourcesExhausted(_))),+ Err(e) => assert!(+ matches!(e.find_root(), DataFusionError::ResourcesExhausted(_)),+ "unexpected error: {e}"+ ),}Ok(())diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.sltindex a4a54cdee2..2be4e173fc 100644--- a/datafusion/sqllogictest/test_files/aggregate.slt+++ b/datafusion/sqllogictest/test_files/aggregate.slt@@ -8789,9 +8789,9 @@ logical_plan15)------------EmptyRelation: rows=1physical_plan01)ProjectionExec: expr=[last_value(a.foo) ORDER BY [a.foo ASC NULLS LAST]@1 as last_value(a.foo) ORDER BY [a.foo ASC NULLS LAST], sum(DISTINCT Int64(1))@2 as sum(DISTINCT Int64(1))]-02)--AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[last_value(a.foo) ORDER BY [a.foo ASC NULLS LAST], sum(DISTINCT Int64(1))], ordering_mode=Sorted+02)--AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[last_value(a.foo) ORDER BY [a.foo ASC NULLS LAST], sum(DISTINCT Int64(1))]03)----RepartitionExec: partitioning=Hash([id@0], 4), input_partitions=5-04)------AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[last_value(a.foo) ORDER BY [a.foo ASC NULLS LAST], sum(DISTINCT Int64(1))], ordering_mode=Sorted+04)------AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[last_value(a.foo) ORDER BY [a.foo ASC NULLS LAST], sum(DISTINCT Int64(1))]05)--------UnionExec06)----------ProjectionExec: expr=[1 as id, 2 as foo]07)------------PlaceholderRowExecdiff --git a/datafusion/sqllogictest/test_files/array_agg.slt b/datafusion/sqllogictest/test_files/array_agg.sltindex b3cd60b9fe..451241756d 100644--- a/datafusion/sqllogictest/test_files/array_agg.slt+++ b/datafusion/sqllogictest/test_files/array_agg.slt@@ -574,9 +574,9 @@ logical_plan15)------------EmptyRelation: rows=1physical_plan01)ProjectionExec: expr=[array_length(array_agg(DISTINCT a.foo)@1) as array_length(array_agg(DISTINCT a.foo)), sum(DISTINCT Int64(1))@2 as sum(DISTINCT Int64(1))]-02)--AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[array_agg(DISTINCT a.foo), sum(DISTINCT Int64(1))], ordering_mode=Sorted+02)--AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[array_agg(DISTINCT a.foo), sum(DISTINCT Int64(1))]03)----RepartitionExec: partitioning=Hash([id@0], 4), input_partitions=5-04)------AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[array_agg(DISTINCT a.foo), sum(DISTINCT Int64(1))], ordering_mode=Sorted+04)------AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[array_agg(DISTINCT a.foo), sum(DISTINCT Int64(1))]05)--------UnionExec06)----------ProjectionExec: expr=[1 as id, CAST(2 AS Int64) as foo]07)------------PlaceholderRowExecdiff --git a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt b/datafusion/sqllogictest/test_files/push_down_filter_regression.sltindex 23013d481f..c327f8092f 100644--- a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt+++ b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt@@ -512,9 +512,9 @@ query TTEXPLAIN SELECT a, count(b) FROM agg_filter_pushdown WHERE a = 'x' GROUP BY a;----physical_plan-01)AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(agg_filter_pushdown.b)], ordering_mode=Sorted+01)AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(agg_filter_pushdown.b)]02)--RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4-03)----AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(agg_filter_pushdown.b)], ordering_mode=Sorted+03)----AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(agg_filter_pushdown.b)]04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=105)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_filter_pushdown.parquet]]}, projection=[a, b], file_type=parquet, predicate=a@0 = x, pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= x AND x <= a_max@1, required_guarantees=[a in (x)]diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.sltindex 9d953f5075..115a010330 100644--- a/datafusion/sqllogictest/test_files/union.slt+++ b/datafusion/sqllogictest/test_files/union.slt@@ -783,7 +783,7 @@ logical_planphysical_plan01)UnionExec02)--ProjectionExec: expr=[Int64(1)@0 as a]-03)----AggregateExec: mode=SinglePartitioned, gby=[1 as Int64(1)], aggr=[], ordering_mode=Sorted+03)----AggregateExec: mode=SinglePartitioned, gby=[1 as Int64(1)], aggr=[]04)------PlaceholderRowExec05)--ProjectionExec: expr=[2 as a]06)----PlaceholderRowExec@@ -811,7 +811,7 @@ logical_planphysical_plan01)UnionExec02)--ProjectionExec: expr=[count(Int64(1))@1 as count, CAST(n@0 AS Int64) as n]-03)----AggregateExec: mode=SinglePartitioned, gby=[n@0 as n], aggr=[count(Int64(1))], ordering_mode=Sorted+03)----AggregateExec: mode=SinglePartitioned, gby=[n@0 as n], aggr=[count(Int64(1))]04)------ProjectionExec: expr=[5 as n]05)--------PlaceholderRowExec06)--ProjectionExec: expr=[1 as count, max(Int64(10))@0 as n]diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.sltindex 6ffe1b4cc0..b0eac5528a 100644--- a/datafusion/sqllogictest/test_files/window.slt+++ b/datafusion/sqllogictest/test_files/window.slt@@ -276,7 +276,7 @@ physical_plan03)----SortExec: expr=[b@0 ASC NULLS LAST], preserve_partitioning=[true]04)------AggregateExec: mode=FinalPartitioned, gby=[b@0 as b], aggr=[max(d.a)]05)--------RepartitionExec: partitioning=Hash([b@0], 4), input_partitions=4-06)----------AggregateExec: mode=Partial, gby=[b@1 as b], aggr=[max(d.a)], ordering_mode=Sorted+06)----------AggregateExec: mode=Partial, gby=[b@1 as b], aggr=[max(d.a)]07)------------UnionExec08)--------------ProjectionExec: expr=[1 as a, aa as b]09)----------------PlaceholderRowExec