Skip to content

Apache DataFusion SQL Query Engine

fix: decorrelate grouping sets that leave out the correlated column when safe (#25781)

namanjain24-sudo290ad15cc842
diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs
index ca2f56c417..0a1db9b212 100644
--- a/datafusion/optimizer/src/decorrelate.rs
+++ b/datafusion/optimizer/src/decorrelate.rs
@@ -36,8 +36,8 @@ use datafusion_expr::utils::{
collect_subquery_cols, conjunction, find_join_exprs, split_conjunction,
};
use datafusion_expr::{
- BinaryExpr, Cast, Distinct, EmptyRelation, Expr, ExprSchemable, Extension, FetchType,
- LogicalPlan, LogicalPlanBuilder, MaterializedCte, Operator, SkipType,
+ Aggregate, BinaryExpr, Cast, Distinct, EmptyRelation, Expr, ExprSchemable, Extension,
+ FetchType, LogicalPlan, LogicalPlanBuilder, MaterializedCte, Operator, SkipType,
UserDefinedLogicalNodeCore, expr, lit,
};
@@ -92,6 +92,14 @@ pub struct PullUpCorrelatedExpr {
/// The list is cleared when the pull up passes a node that can put a NULL
/// back into such a column: an outer join, a union or a grouping set.
pub correlated_filters: Vec<Expr>,
+ /// Columns each node on the path from the subquery's root to the node
+ /// [`Self::f_down`]/[`Self::f_up`] is currently visiting reads in its own
+ /// expressions: one entry per node on that path, pushed in `f_down` and
+ /// popped in `f_up`. When `f_up` visits an `Aggregate`, this holds every
+ /// node strictly above it - its own entry is popped first - so a
+ /// grouping set that omits a correlated column can tell whether
+ /// something above it reads the column the pull up would fill in.
+ column_refs_above: Vec<BTreeSet<Column>>,
}
impl Default for PullUpCorrelatedExpr {
@@ -114,6 +122,7 @@ impl PullUpCorrelatedExpr {
pull_up_having_expr: None,
pulled_up_scalar_agg: false,
correlated_filters: Vec::new(),
+ column_refs_above: Vec::new(),
}
}
@@ -138,6 +147,21 @@ impl PullUpCorrelatedExpr {
}
}
+/// The outcome of trying to fold the pull up's columns into a grouping set
+/// `Aggregate`. See [`PullUpCorrelatedExpr::grouping_sets_pull_up_outcome`].
+#[derive(Debug, PartialEq, Eq)]
+enum GroupingSetsPullUp {
+ /// Every set already groups by the columns; the aggregate's `group_expr`
+ /// stays as it is.
+ NoColumnsToAdd,
+ /// Some set leaves a required column out, but extending every set with it
+ /// changes nothing the rest of the query can observe.
+ SafeToExtend,
+ /// Some set leaves a required column out, and extending every set with it
+ /// would change what the query returns.
+ Unsafe,
+}
+
/// Used to indicate the unmatched rows from the inner(subquery) table after the left out Join
/// This is used to handle [the Count bug]
///
@@ -153,6 +177,15 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
type Node = LogicalPlan;
fn f_down(&mut self, plan: LogicalPlan) -> Result<Transformed<LogicalPlan>> {
+ // Tracks the columns this node's own expressions read, so a
+ // descendant `Aggregate` can later tell what reads above it. Popped
+ // by the matching `f_up` call for this same node.
+ let mut column_refs = BTreeSet::new();
+ for expr in plan.expressions() {
+ column_refs.extend(expr.column_refs().into_iter().cloned());
+ }
+ self.column_refs_above.push(column_refs);
+
// This match is exhaustive on purpose. A correlated filter moves above
// a node only when the node is known to keep the result the same. Add
// each new `LogicalPlan` variant to one of the groups below.
@@ -262,7 +295,12 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
match (limit.get_skip_type()?, fetch) {
(SkipType::Literal(0), FetchType::Literal(_)) => {
// The rewriter does not call `f_down` on the returned
- // node, so do it here
+ // node, so do it here. This call pushes its own
+ // entry onto `column_refs_above`; pop the one this
+ // call pushed for the `Limit` itself first; only one
+ // `f_up` call reaches this tree position once `Limit`
+ // is gone, so only one entry must remain pushed.
+ self.column_refs_above.pop();
let mut t = self.f_down((*limit.input).clone())?;
t.transformed = true;
Ok(t)
@@ -332,6 +370,10 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
}
fn f_up(&mut self, plan: LogicalPlan) -> Result<Transformed<LogicalPlan>> {
+ // Pop this node's own entry, pushed by the matching `f_down` call, so
+ // `column_refs_above` holds only the nodes strictly above `plan`.
+ self.column_refs_above.pop();
+
// A node that null-extends or regroups rows can put a NULL back into
// a column that a correlated filter below it rejected, so the filters
// recorded so far no longer bound what the subquery returns.
@@ -470,42 +512,54 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
&mut local_correlated_cols,
);
- // A grouping set cannot take the columns the pull up adds.
- // `LogicalPlanBuilder::aggregate` cross joins a plain group
- // expression with the sets that are already there, so `ROLLUP(i.k)`,
- // which is `GROUPING SETS ((i.k), ())`, becomes
- // `GROUPING SETS ((i.k), (i.k, i.k))`. The empty set is gone, and
- // with it the grand total row the subquery returns for every outer
- // row, including the rows whose correlated filter matches nothing.
- // The join that replaces the filter cannot bring those rows back,
- // so the subquery stays correlated unless every set already groups
- // by each column the pull up would add.
+ // A grouping set that omits a column the pull up adds either
+ // keeps its sets as they are (every set already groups by the
+ // column), or gains the column in every set, `ROLLUP(i.k)`
+ // (`GROUPING SETS ((i.k), ())`) becoming
+ // `GROUPING SETS ((i.k), (i.k, i.k))`. The latter loses the
+ // empty set, and with it the grand total row the subquery
+ // returns for every outer row, including the rows whose
+ // correlated filter matches nothing, so it is only safe when
+ // no set is empty and nothing reads the column the fill
+ // changes from `NULL` to the correlated value.
let mut missing_exprs = if aggregate
.group_expr
.iter()
.any(|expr| matches!(expr, Expr::GroupingSet(_)))
{
- if self.grouping_sets_cover_pull_up_cols(
- &aggregate.group_expr,
- &local_correlated_cols,
- ) {
- // Every set already groups by them, so the sets stay as
- // they are. Adding the columns again would repeat them
- // inside every set.
- aggregate.group_expr.to_vec()
- } else {
- self.can_pull_up = false;
- // The rewrite still runs, the same way the
- // `can_pull_over_aggregation` case above does. The callers
- // read `can_pull_up` only after the whole rewrite has
- // finished, and the nodes above this one still expect the
- // pulled up columns in its output, so leaving them out here
- // would fail the rewrite with a schema error instead. They
- // drop this plan and keep the correlated subquery.
- self.collect_missing_exprs(
- &aggregate.group_expr,
- &local_correlated_cols,
- )?
+ match self
+ .grouping_sets_pull_up_outcome(aggregate, &local_correlated_cols)
+ {
+ GroupingSetsPullUp::NoColumnsToAdd => {
+ // Every set already groups by them, so the sets stay as
+ // they are. Adding the columns again would repeat them
+ // inside every set.
+ aggregate.group_expr.to_vec()
+ }
+ GroupingSetsPullUp::SafeToExtend => {
+ // No set is empty and nothing reads the columns the
+ // fill changes from `NULL` to the correlated value,
+ // so extending every set is invisible to the rest of
+ // the query.
+ self.collect_missing_exprs(
+ &aggregate.group_expr,
+ &local_correlated_cols,
+ )?
+ }
+ GroupingSetsPullUp::Unsafe => {
+ self.can_pull_up = false;
+ // The rewrite still runs, the same way the
+ // `can_pull_over_aggregation` case above does. The callers
+ // read `can_pull_up` only after the whole rewrite has
+ // finished, and the nodes above this one still expect the
+ // pulled up columns in its output, so leaving them out here
+ // would fail the rewrite with a schema error instead. They
+ // drop this plan and keep the correlated subquery.
+ self.collect_missing_exprs(
+ &aggregate.group_expr,
+ &local_correlated_cols,
+ )?
+ }
}
} else {
// add missing columns to Aggregation's group expressions
@@ -651,26 +705,40 @@ impl PullUpCorrelatedExpr {
Ok(Transformed::yes(plan.recompute_schema()?))
}
- /// Whether the pull up can add its columns to `group_expr` without changing
- /// what the aggregate returns.
+ /// Whether the pull up can add its columns to a grouping set `Aggregate`,
+ /// and whether it needs to.
///
- /// `true` when `group_expr` holds no grouping set, and when every set of every
- /// grouping set it holds already groups by each column
- /// [`Self::collect_missing_exprs`] would add. In the second case the pull up
- /// adds nothing and the aggregate keeps the sets it has.
+ /// [`GroupingSetsPullUp::NoColumnsToAdd`] when every set of every grouping
+ /// set already groups by each column [`Self::collect_missing_exprs`]
+ /// would add: the pull up adds nothing and the aggregate keeps the sets
+ /// it has.
///
- /// `ROLLUP` and `CUBE` always contain the empty set, which yields a row for
- /// outer rows the correlated filter matches nothing for, so they are only safe
- /// when there is nothing to add.
+ /// Otherwise some set leaves a required column out, which fills that
+ /// column with `NULL` in the set's rows. Adding the column to every set
+ /// (`LogicalPlanBuilder::aggregate` cross joins a plain group expression
+ /// with the sets that are already there) gives it a value instead. This
+ /// is only invisible to the rest of the query when:
+ /// - no set is empty. `ROLLUP(i.k)`, `GROUPING SETS ((i.k), ())`, always
+ /// holds one, and turns into `GROUPING SETS ((i.k), (i.k, i.k))`,
+ /// losing the empty set and the grand total row it yields for every
+ /// outer row, including the rows the correlated filter matches
+ /// nothing for. The join that replaces the filter cannot bring that
+ /// row back.
+ /// - the aggregate does not compute `GROUPING`/`GROUPING_ID`, which
+ /// reads whether a row's set groups by a column.
+ /// - nothing above the aggregate reads the column, which
+ /// `column_refs_above` tracks as `f_down`/`f_up` push and pop a node's
+ /// own columns while they walk down to and back up from it.
///
- /// A non-empty set that leaves a column out fills it with NULL. Adding the
- /// column would give it a value instead, which a `HAVING` or a projection
- /// above the aggregate can read, so such a set is rejected as well.
- fn grouping_sets_cover_pull_up_cols(
+ /// When one of these fails, [`GroupingSetsPullUp::Unsafe`] is returned:
+ /// the columns are still added, so the schema the nodes above expect
+ /// stays correct, but the caller must give up on decorrelating.
+ fn grouping_sets_pull_up_outcome(
&self,
- group_expr: &[Expr],
+ aggregate: &Aggregate,
correlated_subquery_cols: &BTreeSet<Column>,
- ) -> bool {
+ ) -> GroupingSetsPullUp {
+ let group_expr = &aggregate.group_expr;
let grouping_sets = group_expr
.iter()
.filter_map(|expr| match expr {
@@ -679,7 +747,7 @@ impl PullUpCorrelatedExpr {
})
.collect::<Vec<_>>();
if grouping_sets.is_empty() {
- return true;
+ return GroupingSetsPullUp::NoColumnsToAdd;
}
// The same columns `collect_missing_exprs` appends: the correlated columns
@@ -695,18 +763,40 @@ impl PullUpCorrelatedExpr {
.any(|expr| matches!(expr, Expr::Column(c) if c == *col))
});
if required_cols.is_empty() {
- return true;
+ return GroupingSetsPullUp::NoColumnsToAdd;
}
- grouping_sets.iter().all(|grouping_set| match grouping_set {
- GroupingSet::Rollup(_) | GroupingSet::Cube(_) => false,
- GroupingSet::GroupingSets(sets) => sets.iter().all(|set| {
- required_cols.iter().all(|col| {
- set.iter()
- .any(|expr| matches!(expr, Expr::Column(c) if c == *col))
- })
- }),
- })
+ let already_covered =
+ grouping_sets.iter().all(|grouping_set| match grouping_set {
+ GroupingSet::Rollup(_) | GroupingSet::Cube(_) => false,
+ GroupingSet::GroupingSets(sets) => sets.iter().all(|set| {
+ required_cols.iter().all(|col| {
+ set.iter()
+ .any(|expr| matches!(expr, Expr::Column(c) if c == *col))
+ })
+ }),
+ });
+ if already_covered {
+ return GroupingSetsPullUp::NoColumnsToAdd;
+ }
+
+ let has_empty_set = grouping_sets.iter().any(|grouping_set| match grouping_set {
+ GroupingSet::Rollup(_) | GroupingSet::Cube(_) => true,
+ GroupingSet::GroupingSets(sets) => sets.iter().any(|set| set.is_empty()),
+ });
+ // `GROUPING`/`GROUPING_ID` are resolved to bit tests on `__grouping_id`
+ // above the aggregate by `ResolveGroupingFunction` before this rule
+ // runs, so `aggregate.aggr_expr` never holds a `GROUPING` call by the
+ // time this is reached; adding a column shifts those bits instead.
+ let unsafe_read = self.column_refs_above.iter().flatten().any(|col| {
+ col.name == Aggregate::INTERNAL_GROUPING_ID || required_cols.contains(col)
+ });
+
+ if has_empty_set || unsafe_read {
+ GroupingSetsPullUp::Unsafe
+ } else {
+ GroupingSetsPullUp::SafeToExtend
+ }
}
fn collect_missing_exprs(
diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt
index 09f648df27..39b3961c82 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -2907,6 +2907,10 @@ DROP TABLE metrics;
# `GROUPING SETS ((k), ())`, turns into `GROUPING SETS ((k), (k, k))`. The empty
# set is gone, and with it the grand total row the subquery returns for every
# outer row, including the rows whose filter matches nothing.
+#
+# Follow-up (#25708): a non-empty set that leaves the correlated column out is
+# safe to extend when nothing above the aggregate reads that column, so those
+# queries decorrelate again below.
statement ok
CREATE TABLE gs_outer(k INT) AS VALUES (1), (2), (NULL), (4), (5);
@@ -2927,14 +2931,36 @@ SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k
# A set that leaves out the correlated column fills it with NULL. The pull up
# would turn `(j)` into `(j, k)`, and `k` would then carry a value in the rows
-# where the set fills it with NULL. Anything above the aggregate that reads `k`
-# sees the difference, so the subquery stays correlated.
-#
-# Known limitation: when nothing reads `k`, as here, the pull up was correct
-# before this guard and the query now fails to plan. Telling the two cases apart
-# needs the correlated column added to each set under an alias.
+# where the set fills it with NULL. Nothing above this `EXISTS` reads `k`
+# though (it only checks for a row), so the difference is invisible and the
+# subquery still decorrelates.
+query IB
+SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY GROUPING SETS ((gs_inner.k), (gs_inner.j))) FROM gs_outer ORDER BY gs_outer.k;
+----
+1 true
+2 true
+4 false
+5 true
+NULL false
+
+# A filter above the grouping set `Aggregate`, through another `Aggregate`
+# stacked on top of it, reads the NULL filled column. `walk` must not stop at
+# the first `Aggregate` it finds (the outer `count(*)`) without recursing into
+# its input, or it never sees that read and wrongly decorrelates.
+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery
+SELECT gs_outer.k, (SELECT count(*) FROM (SELECT gs_inner.k AS kk FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY GROUPING SETS ((gs_inner.k), (gs_inner.j))) t WHERE t.kk IS NULL) FROM gs_outer ORDER BY gs_outer.k;
+
+# The same read, but in a later join input rather than directly above the
+# grouping set `Aggregate`. `walk` must visit every input, not stop once one
+# of them turns up a grouping set `Aggregate` below it.
statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists
-SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY GROUPING SETS ((gs_inner.k), (gs_inner.j))) FROM gs_outer;
+SELECT gs_outer.k, EXISTS (SELECT 1 FROM (SELECT count(*) AS c FROM gs_inner) a JOIN (SELECT gs_inner.k FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY GROUPING SETS ((gs_inner.k), (gs_inner.j))) b ON a.c > 0 WHERE b.k IS NULL) FROM gs_outer ORDER BY gs_outer.k;
+
+# The same read, above a second, nested grouping set `Aggregate` stacked on
+# top of the first. `walk` must keep walking through a grouping set
+# `Aggregate` too, not only a plain one.
+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists
+SELECT gs_outer.k, EXISTS (SELECT 1 FROM (SELECT gs_inner.k AS kk, gs_inner.j AS jj FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY GROUPING SETS ((gs_inner.k), (gs_inner.j))) t GROUP BY GROUPING SETS ((t.kk), (t.kk, t.jj)) HAVING t.kk IS NULL) FROM gs_outer ORDER BY gs_outer.k;
# The same sets with a HAVING that reads the NULL filled column. For k = 1 the
# `(j)` set yields the row `(NULL, 10)`, which passes the HAVING, so EXISTS is
@@ -2943,6 +2969,32 @@ SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k
statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists
SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY GROUPING SETS ((gs_inner.k), (gs_inner.j)) HAVING gs_inner.k IS NULL) FROM gs_outer;
+# The same sets with a `GROUPING(k)` read out of the aggregate itself: for the
+# `(j)` set it is 1 (k is not one of its columns) before the pull up, and 0
+# (k is now one of its columns in every set) after, so the subquery stays
+# correlated even though nothing above the aggregate reads `k` directly.
+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery
+SELECT gs_outer.k, (SELECT GROUPING(gs_inner.k) FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY GROUPING SETS ((gs_inner.k), (gs_inner.j)) LIMIT 1) FROM gs_outer;
+
+# `GROUPING`/`GROUPING_ID` are rewritten by `ResolveGroupingFunction` into bit
+# tests on `__grouping_id` in a node above the aggregate before this rule
+# runs, so the aggregate's own `aggr_expr` never holds a raw `GROUPING` call.
+# Reading `GROUPING(j)` out of the `(j)`/`(x)` sets, as a HAVING or through an
+# outer projection, reads `__grouping_id` above the aggregate without reading
+# `j` or `x` themselves, so this is only caught by looking for
+# `__grouping_id` in `column_refs_above_aggregate` directly.
+statement ok
+CREATE TABLE gs_inner3(k INT, j INT, x INT) AS VALUES (1, 10, 100), (NULL, 20, 200), (5, 30, 300), (2, 40, 400);
+
+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists
+SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner3 WHERE gs_inner3.k = gs_outer.k GROUP BY GROUPING SETS ((gs_inner3.j), (gs_inner3.x)) HAVING GROUPING(gs_inner3.j) = 1) FROM gs_outer ORDER BY gs_outer.k;
+
+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery
+SELECT gs_outer.k, (SELECT count(*) FROM (SELECT GROUPING(gs_inner3.j) AS g FROM gs_inner3 WHERE gs_inner3.k = gs_outer.k GROUP BY GROUPING SETS ((gs_inner3.j), (gs_inner3.x))) t WHERE t.g = 1) FROM gs_outer ORDER BY gs_outer.k;
+
+statement ok
+DROP TABLE gs_inner3;
+
# The correlated column does not have to appear in the grouping set at all.
statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists
SELECT gs_outer.k, EXISTS (SELECT 1 FROM gs_inner WHERE gs_inner.k = gs_outer.k GROUP BY ROLLUP(gs_inner.j)) FROM gs_outer;