Apache DataFusion SQL Query Engine
fix: keep a correlated filter below an aggregate with a grouping set (#25529)
namanjain24-sudofb42cf5911fa
diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rsindex 0c37f00b64..a288ded88b 100644--- a/datafusion/optimizer/src/decorrelate.rs+++ b/datafusion/optimizer/src/decorrelate.rs@@ -28,7 +28,7 @@ use datafusion_common::tree_node::{use datafusion_common::{Column, DFSchemaRef, HashMap, Result, ScalarValue, assert_or_internal_err, plan_err,};-use datafusion_expr::expr::Alias;+use datafusion_expr::expr::{Alias, GroupingSet};use datafusion_expr::simplify::SimplifyContext;use datafusion_expr::utils::{collect_subquery_cols, conjunction, find_join_exprs, split_conjunction,@@ -299,11 +299,51 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {&self.correlated_subquery_cols_map,&mut local_correlated_cols,);- // add missing columns to Aggregation's group expressions- let mut missing_exprs = self.collect_missing_exprs(- &aggregate.group_expr,- &local_correlated_cols,- )?;++ // A grouping set cannot take the columns the pull up adds.+ // `LogicalPlanBuilder::aggregate` cross joins a plain group+ // expression with the sets that are already there, so `ROLLUP(i.k)`,+ // which is `GROUPING SETS ((i.k), ())`, becomes+ // `GROUPING SETS ((i.k), (i.k, i.k))`. The empty set is gone, and+ // with it the grand total row the subquery returns for every outer+ // row, including the rows whose correlated filter matches nothing.+ // The join that replaces the filter cannot bring those rows back,+ // so the subquery stays correlated unless every set already groups+ // by each column the pull up would add.+ let mut missing_exprs = if aggregate+ .group_expr+ .iter()+ .any(|expr| matches!(expr, Expr::GroupingSet(_)))+ {+ if self.grouping_sets_cover_pull_up_cols(+ &aggregate.group_expr,+ &local_correlated_cols,+ ) {+ // Every set already groups by them, so the sets stay as+ // they are. Adding the columns again would repeat them+ // inside every set.+ aggregate.group_expr.to_vec()+ } else {+ self.can_pull_up = false;+ // The rewrite still runs, the same way the+ // `can_pull_over_aggregation` case above does. The callers+ // read `can_pull_up` only after the whole rewrite has+ // finished, and the nodes above this one still expect the+ // pulled up columns in its output, so leaving them out here+ // would fail the rewrite with a schema error instead. They+ // drop this plan and keep the correlated subquery.+ self.collect_missing_exprs(+ &aggregate.group_expr,+ &local_correlated_cols,+ )?+ }+ } else {+ // add missing columns to Aggregation's group expressions+ self.collect_missing_exprs(+ &aggregate.group_expr,+ &local_correlated_cols,+ )?+ };// if the original group expressions are empty, need to handle the Count buglet mut expr_result_map_for_count_bug = HashMap::new();@@ -404,6 +444,64 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {}impl PullUpCorrelatedExpr {+ /// Whether the pull up can add its columns to `group_expr` without changing+ /// what the aggregate returns.+ ///+ /// `true` when `group_expr` holds no grouping set, and when every set of every+ /// grouping set it holds already groups by each column+ /// [`Self::collect_missing_exprs`] would add. In the second case the pull up+ /// adds nothing and the aggregate keeps the sets it has.+ ///+ /// `ROLLUP` and `CUBE` always contain the empty set, which yields a row for+ /// outer rows the correlated filter matches nothing for, so they are only safe+ /// when there is nothing to add.+ ///+ /// A non-empty set that leaves a column out fills it with NULL. Adding the+ /// column would give it a value instead, which a `HAVING` or a projection+ /// above the aggregate can read, so such a set is rejected as well.+ fn grouping_sets_cover_pull_up_cols(+ &self,+ group_expr: &[Expr],+ correlated_subquery_cols: &BTreeSet<Column>,+ ) -> bool {+ let grouping_sets = group_expr+ .iter()+ .filter_map(|expr| match expr {+ Expr::GroupingSet(grouping_set) => Some(grouping_set),+ _ => None,+ })+ .collect::<Vec<_>>();+ if grouping_sets.is_empty() {+ return true;+ }++ // The same columns `collect_missing_exprs` appends: the correlated columns+ // and the columns of a pulled up HAVING, minus the ones `group_expr`+ // already lists on their own, which it leaves alone.+ let mut required_cols = correlated_subquery_cols.iter().collect::<BTreeSet<_>>();+ if let Some(pull_up_having) = &self.pull_up_having_expr {+ required_cols.extend(pull_up_having.column_refs());+ }+ required_cols.retain(|col| {+ !group_expr+ .iter()+ .any(|expr| matches!(expr, Expr::Column(c) if c == *col))+ });+ if required_cols.is_empty() {+ return true;+ }++ grouping_sets.iter().all(|grouping_set| match grouping_set {+ GroupingSet::Rollup(_) | GroupingSet::Cube(_) => false,+ GroupingSet::GroupingSets(sets) => sets.iter().all(|set| {+ required_cols.iter().all(|col| {+ set.iter()+ .any(|expr| matches!(expr, Expr::Column(c) if c == *col))+ })+ }),+ })+ }+fn collect_missing_exprs(&self,exprs: &[Expr],diff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rsindex 135d1fc7f5..cd5ca84f66 100644--- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs+++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs@@ -772,7 +772,9 @@ mod tests {use crate::assert_optimized_plan_eq_display_indent_snapshot;use arrow::datatypes::{DataType, Field, Schema};use datafusion_expr::builder::table_source;- use datafusion_expr::{and, binary_expr, col, out_ref_col, table_scan};+ use datafusion_expr::{+ and, binary_expr, col, cube, grouping_set, out_ref_col, rollup, table_scan,+ };macro_rules! assert_optimized_plan_equal {(@@ -822,6 +824,138 @@ mod tests {optimizer.optimize(plan, &crate::OptimizerContext::new(), |_, _| {})}+ /// A grouping set subquery for the tests below: `SELECT c FROM <name> WHERE+ /// c = test.c GROUP BY <group_expr>`.+ fn correlated_grouping_set_subquery(+ name: &str,+ group_expr: Expr,+ ) -> Result<Arc<LogicalPlan>> {+ Ok(Arc::new(+ LogicalPlanBuilder::from(test_table_scan_with_name(name)?)+ .filter(+ col(format!("{name}.c")).eq(out_ref_col(DataType::UInt32, "test.c")),+ )?+ .aggregate(vec![group_expr], Vec::<Expr>::new())?+ .project(vec![col(format!("{name}.c"))])?+ .build()?,+ ))+ }++ /// `ROLLUP(c)` is `GROUPING SETS ((c), ())`. Adding the correlated column to+ /// every set drops the empty one, so the subquery is left correlated.+ /// <https://github.com/apache/datafusion/issues/25519>+ #[test]+ fn exists_subquery_with_rollup_is_not_decorrelated() -> Result<()> {+ let subquery = correlated_grouping_set_subquery("sq", rollup(vec![col("sq.c")]))?;+ let plan = LogicalPlanBuilder::from(test_table_scan()?)+ .filter(exists(subquery))?+ .project(vec![col("test.b")])?+ .build()?;++ assert_optimized_plan_equal!(+ plan,+ @r"+ Projection: test.b [b:UInt32]+ Filter: EXISTS (<subquery>) [a:UInt32, b:UInt32, c:UInt32]+ Subquery: [c:UInt32;N]+ Projection: sq.c [c:UInt32;N]+ Aggregate: groupBy=[[ROLLUP (sq.c)]], aggr=[[]] [c:UInt32;N, __grouping_id:UInt8]+ Filter: sq.c = outer_ref(test.c) [a:UInt32, b:UInt32, c:UInt32]+ TableScan: sq [a:UInt32, b:UInt32, c:UInt32]+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]+ "+ )+ }++ /// `CUBE(c)` holds the empty set for the same reason. The correlation is on+ /// `a` rather than on the `IN` key, so it stays a filter of its own instead+ /// of being folded into the `IN` predicate.+ /// <https://github.com/apache/datafusion/issues/25519>+ #[test]+ fn in_subquery_with_cube_is_not_decorrelated() -> Result<()> {+ let subquery = Arc::new(+ LogicalPlanBuilder::from(test_table_scan_with_name("sq")?)+ .filter(col("sq.a").eq(out_ref_col(DataType::UInt32, "test.a")))?+ .aggregate(vec![cube(vec![col("sq.c")])], Vec::<Expr>::new())?+ .project(vec![col("sq.c")])?+ .build()?,+ );+ let plan = LogicalPlanBuilder::from(test_table_scan()?)+ .filter(in_subquery(col("test.c"), subquery))?+ .project(vec![col("test.b")])?+ .build()?;++ assert_optimized_plan_equal!(+ plan,+ @r"+ Projection: test.b [b:UInt32]+ Filter: test.c IN (<subquery>) [a:UInt32, b:UInt32, c:UInt32]+ Subquery: [c:UInt32;N]+ Projection: sq.c [c:UInt32;N]+ Aggregate: groupBy=[[CUBE (sq.c)]], aggr=[[]] [c:UInt32;N, __grouping_id:UInt8]+ Filter: sq.a = outer_ref(test.a) [a:UInt32, b:UInt32, c:UInt32]+ TableScan: sq [a:UInt32, b:UInt32, c:UInt32]+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]+ "+ )+ }++ /// A set that groups by another column does not carry the correlated one.+ /// <https://github.com/apache/datafusion/issues/25519>+ #[test]+ fn exists_subquery_with_partial_grouping_set_is_not_decorrelated() -> Result<()> {+ let subquery = correlated_grouping_set_subquery(+ "sq",+ grouping_set(vec![vec![col("sq.c")], vec![col("sq.b")]]),+ )?;+ let plan = LogicalPlanBuilder::from(test_table_scan()?)+ .filter(exists(subquery))?+ .project(vec![col("test.b")])?+ .build()?;++ assert_optimized_plan_equal!(+ plan,+ @r"+ Projection: test.b [b:UInt32]+ Filter: EXISTS (<subquery>) [a:UInt32, b:UInt32, c:UInt32]+ Subquery: [c:UInt32;N]+ Projection: sq.c [c:UInt32;N]+ Aggregate: groupBy=[[GROUPING SETS ((sq.c), (sq.b))]], aggr=[[]] [c:UInt32;N, b:UInt32;N, __grouping_id:UInt8]+ Filter: sq.c = outer_ref(test.c) [a:UInt32, b:UInt32, c:UInt32]+ TableScan: sq [a:UInt32, b:UInt32, c:UInt32]+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]+ "+ )+ }++ /// Every set already groups by the correlated column, so the pull up adds+ /// nothing and the subquery decorrelates as it did before.+ /// <https://github.com/apache/datafusion/issues/25519>+ #[test]+ fn exists_subquery_with_covering_grouping_set_is_decorrelated() -> Result<()> {+ let subquery = correlated_grouping_set_subquery(+ "sq",+ grouping_set(vec![vec![col("sq.c")], vec![col("sq.c"), col("sq.b")]]),+ )?;+ let plan = LogicalPlanBuilder::from(test_table_scan()?)+ .filter(exists(subquery))?+ .project(vec![col("test.b")])?+ .build()?;++ assert_optimized_plan_equal!(+ plan,+ @r"+ Projection: test.b [b:UInt32]+ LeftSemi Join: Filter: __correlated_sq_1.c = test.c [a:UInt32, b:UInt32, c:UInt32]+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]+ SubqueryAlias: __correlated_sq_1 [c:UInt32;N]+ Projection: sq.c [c:UInt32;N]+ Aggregate: groupBy=[[GROUPING SETS ((sq.c), (sq.c, sq.b))]], aggr=[[]] [c:UInt32;N, b:UInt32;N, __grouping_id:UInt8]+ TableScan: sq [a:UInt32, b:UInt32, c:UInt32]+ "+ )+ }+/// Test for several IN subquery expressions#[test]fn in_subquery_multiple() -> Result<()> {diff --git a/datafusion/optimizer/src/scalar_subquery_to_join.rs b/datafusion/optimizer/src/scalar_subquery_to_join.rsindex 44011a125b..ed5e0a6def 100644--- a/datafusion/optimizer/src/scalar_subquery_to_join.rs+++ b/datafusion/optimizer/src/scalar_subquery_to_join.rs@@ -445,7 +445,7 @@ mod tests {use datafusion_expr::test::function_stub::sum;use crate::assert_optimized_plan_eq_display_indent_snapshot;- use datafusion_expr::{Between, col, expr, out_ref_col, scalar_subquery};+ use datafusion_expr::{Between, col, expr, out_ref_col, rollup, scalar_subquery};use datafusion_functions_aggregate::min_max::{max, min};macro_rules! assert_optimized_plan_equal {@@ -462,6 +462,46 @@ mod tests {}};}+ /// A correlated scalar subquery whose aggregate uses `ROLLUP` keeps its+ /// correlation: the empty set yields a row for outer rows the filter matches+ /// nothing for, and the join that would replace the filter cannot produce it.+ /// <https://github.com/apache/datafusion/issues/25519>+ #[test]+ fn scalar_subquery_with_rollup_is_not_decorrelated() -> Result<()> {+ let sq = Arc::new(+ LogicalPlanBuilder::from(scan_tpch_table("orders"))+ .filter(+ col("orders.o_custkey")+ .eq(out_ref_col(DataType::Int64, "customer.c_custkey")),+ )?+ .aggregate(+ vec![rollup(vec![col("orders.o_custkey")])],+ vec![max(col("orders.o_custkey"))],+ )?+ .project(vec![max(col("orders.o_custkey"))])?+ .build()?,+ );++ let plan = LogicalPlanBuilder::from(scan_tpch_table("customer"))+ .filter(col("customer.c_custkey").eq(scalar_subquery(sq)))?+ .project(vec![col("customer.c_custkey")])?+ .build()?;++ assert_optimized_plan_equal!(+ plan,+ @r"+ Projection: customer.c_custkey [c_custkey:Int64]+ Filter: customer.c_custkey = (<subquery>) [c_custkey:Int64, c_name:Utf8]+ Subquery: [max(orders.o_custkey):Int64;N]+ Projection: max(orders.o_custkey) [max(orders.o_custkey):Int64;N]+ Aggregate: groupBy=[[ROLLUP (orders.o_custkey)]], aggr=[[max(orders.o_custkey)]] [o_custkey:Int64;N, __grouping_id:UInt8, max(orders.o_custkey):Int64;N]+ Filter: orders.o_custkey = outer_ref(customer.c_custkey) [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]+ TableScan: orders [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]+ TableScan: customer [c_custkey:Int64, c_name:Utf8]+ "+ )+ }+/// Test multiple correlated subqueries#[test]fn multiple_subqueries() -> Result<()> {diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex c208d7ab19..f383f66176 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -2744,3 +2744,107 @@ b 400statement okDROP TABLE metrics;++# Regression test for #25519: a correlated filter that sits below an aggregate+# with a grouping set must not be pulled above it. The pull up adds the+# correlated column to every set, so `ROLLUP(k)`, which is+# `GROUPING SETS ((k), ())`, turns into `GROUPING SETS ((k), (k, k))`. The empty+# set is gone, and with it the grand total row the subquery returns for every+# outer row, including the rows whose filter matches nothing.+statement ok+CREATE TABLE gs_outer(k INT) AS VALUES (1), (2), (NULL), (4), (5);++statement ok+CREATE TABLE gs_inner(k INT, j INT) AS VALUES (1, 10), (NULL, 20), (5, 30), (2, 40);++# ROLLUP holds the empty set, so the subquery stays correlated.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists+SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY ROLLUP(gs_inner.k)) FROM gs_outer;++# So does CUBE.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists+SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY CUBE(gs_inner.k)) FROM gs_outer;++# And an explicit grouping set that lists the empty set.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists+SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY GROUPING SETS ((gs_inner.k), ())) FROM gs_outer;++# A set that leaves out the correlated column fills it with NULL. The pull up+# would turn `(j)` into `(j, k)`, and `k` would then carry a value in the rows+# where the set fills it with NULL. Anything above the aggregate that reads `k`+# sees the difference, so the subquery stays correlated.+#+# Known limitation: when nothing reads `k`, as here, the pull up was correct+# before this guard and the query now fails to plan. Telling the two cases apart+# needs the correlated column added to each set under an alias.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists+SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY GROUPING SETS ((gs_inner.k), (gs_inner.j))) FROM gs_outer;++# The same sets with a HAVING that reads the NULL filled column. For k = 1 the+# `(j)` set yields the row `(NULL, 10)`, which passes the HAVING, so EXISTS is+# true; with `(j, k)` that row holds k = 1 and is filtered out. On main this+# query answers false for every row.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists+SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY GROUPING SETS ((gs_inner.k), (gs_inner.j)) HAVING gs_inner.k IS NULL) FROM gs_outer;++# The correlated column does not have to appear in the grouping set at all.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists+SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY ROLLUP(gs_inner.j)) FROM gs_outer;++# The `IN` form of the same subquery.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression InSubquery+SELECT gs_outer.k, gs_outer.k IN (SELECT gs_inner.k FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY ROLLUP(gs_inner.k)) FROM gs_outer;++# `NOT EXISTS` too.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists+SELECT gs_outer.k, NOT EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY ROLLUP(gs_inner.k)) FROM gs_outer;++# And a correlated scalar subquery.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery+SELECT gs_outer.k, (SELECT count(*) FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY ROLLUP(gs_inner.k) LIMIT 1) FROM gs_outer;++# When every set already groups by the correlated column the pull up adds+# nothing, so the subquery still decorrelates and keeps its results.+query IB+SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY GROUPING SETS ((gs_inner.k))) FROM gs_outer ORDER BY gs_outer.k;+----+1 true+2 true+4 false+5 true+NULL false++query IB+SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY GROUPING SETS ((gs_inner.k), (gs_inner.k, gs_inner.j))) FROM gs_outer ORDER BY gs_outer.k;+----+1 true+2 true+4 false+5 true+NULL false++# A plain GROUP BY is unaffected.+query IB+SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY gs_inner.k) FROM gs_outer ORDER BY gs_outer.k;+----+1 true+2 true+4 false+5 true+NULL false++# An uncorrelated subquery keeps its grouping set.+query IB+SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner GROUP BY ROLLUP(gs_inner.k)) FROM gs_outer ORDER BY gs_outer.k;+----+1 true+2 true+4 true+5 true+NULL true++statement ok+DROP TABLE gs_outer;++statement ok+DROP TABLE gs_inner;