Skip to content

Apache DataFusion SQL Query Engine

fix: keep a LIMIT inside EXISTS that an operator above it can empty (#26640)

RAGHVENDRA KUMAR YADAV1aa6c294bd01
diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs
index e1c6235f31..5143ac87c1 100644
--- a/datafusion/optimizer/src/decorrelate.rs
+++ b/datafusion/optimizer/src/decorrelate.rs
@@ -97,6 +97,12 @@ pub struct PullUpCorrelatedExpr {
/// grouping set that omits a correlated column can tell whether
/// something above it reads the column the pull up would fill in.
column_refs_above: Vec<BTreeSet<Column>>,
+ /// For each node on that same path, whether it returns rows based only on
+ /// whether its input has rows (see [`depends_only_on_input_having_rows`]). One
+ /// entry per node, pushed and popped with [`Self::column_refs_above`].
+ /// `f_down`'s `Limit` arm reads it: a `Limit` is only dropped from an
+ /// `EXISTS` subquery when this holds for every node above it.
+ depends_only_on_input_having_rows_above: Vec<bool>,
}
impl Default for PullUpCorrelatedExpr {
@@ -121,6 +127,7 @@ impl PullUpCorrelatedExpr {
pulled_up_scalar_agg: false,
correlated_filters: Vec::new(),
column_refs_above: Vec::new(),
+ depends_only_on_input_having_rows_above: Vec::new(),
}
}
@@ -183,6 +190,8 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
column_refs.extend(expr.column_refs().into_iter().cloned());
}
self.column_refs_above.push(column_refs);
+ self.depends_only_on_input_having_rows_above
+ .push(depends_only_on_input_having_rows(&plan));
// This match is exhaustive on purpose. A correlated filter moves above
// a node only when the node is known to keep the result the same. Add
@@ -268,9 +277,8 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
if !self.exists_sub_query {
return Ok(self.stop_pull_up(plan));
}
- // Only emptiness matters for EXISTS, so remove a limit that
- // cannot make its input empty and replace one that always does
- // with an empty relation.
+ // A `LIMIT 0` always gives an empty input, whatever is
+ // above it, so it becomes an empty relation.
let fetch = limit.get_fetch_type()?;
if matches!(fetch, FetchType::Literal(Some(0))) {
return Ok(Transformed::yes(LogicalPlan::EmptyRelation(
@@ -280,6 +288,20 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
},
)));
}
+ // `EXISTS` asks whether the whole subquery returns a row, not
+ // whether this `Limit` does. If a node above the `Limit` looks
+ // at which rows it gets, such as a `Filter` or the `HAVING` in
+ // https://github.com/apache/datafusion/issues/26629, removing
+ // the `Limit` hands it different rows and can change the
+ // answer, so the `Limit` has to stay.
+ let (_, above) = self
+ .depends_only_on_input_having_rows_above
+ .split_last()
+ .expect("f_down pushed this Limit's own entry");
+ if !above.iter().all(|follows| *follows) {
+ return Ok(self.stop_pull_up(plan));
+ }
+
match (limit.get_skip_type()?, fetch) {
(SkipType::Literal(0), FetchType::Literal(_)) => {
// The rewriter does not call `f_down` on the returned
@@ -289,6 +311,7 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
// `f_up` call reaches this tree position once `Limit`
// is gone, so only one entry must remain pushed.
self.column_refs_above.pop();
+ self.depends_only_on_input_having_rows_above.pop();
let mut t = self.f_down((*limit.input).clone())?;
t.transformed = true;
Ok(t)
@@ -364,6 +387,7 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
// Pop this node's own entry, pushed by the matching `f_down` call, so
// `column_refs_above` holds only the nodes strictly above `plan`.
self.column_refs_above.pop();
+ self.depends_only_on_input_having_rows_above.pop();
// A node that null-extends or regroups rows can put a NULL back into
// a column that a correlated filter below it rejected, so the filters
@@ -897,6 +921,43 @@ impl PullUpCorrelatedExpr {
}
}
+/// Whether `plan` returns rows based only on whether its input has rows, and
+/// not on which rows those are.
+///
+/// This is what lets `EXISTS` drop a `Limit` with no skip and a non-zero
+/// fetch. Such a `Limit` returns rows exactly when its input has rows. If
+/// every node above it also only depends on whether it gets rows, the whole
+/// subquery returns rows exactly when the `Limit`'s input has rows, with or
+/// without the `Limit`, so removing it cannot change the `EXISTS` result.
+///
+/// It holds for:
+/// - `Projection`, `SubqueryAlias`, `Sort`, `Window` and `Repartition`, which
+/// return rows whenever their input has rows (a `Sort` with a fetch of 0
+/// returns none either way)
+/// - `DISTINCT` and a grouped `Aggregate`, which return at least one row for a
+/// non-empty input and none for an empty one
+/// - an `Aggregate` without grouping, which always returns one row
+/// - a materialized CTE, which returns its continuation's rows
+///
+/// It does not hold for a `Filter`, which keeps or removes rows by their
+/// values, so the rows the `Limit` kept decide whether any survive. A `Join`
+/// and an `Unnest` can also turn rows into none. Every other plan, including a
+/// user-defined `Extension`, is treated as not holding: the `Limit` stays and
+/// the subquery is not decorrelated.
+fn depends_only_on_input_having_rows(plan: &LogicalPlan) -> bool {
+ matches!(
+ plan,
+ LogicalPlan::Projection(_)
+ | LogicalPlan::SubqueryAlias(_)
+ | LogicalPlan::Sort(_)
+ | LogicalPlan::Window(_)
+ | LogicalPlan::Repartition(_)
+ | LogicalPlan::Distinct(Distinct::All(_))
+ | LogicalPlan::Aggregate(_)
+ | LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(_))
+ )
+}
+
/// Whether `plan` can output a NULL in a column that a filter below it keeps
/// free of NULLs: an outer join fills the columns of its unmatched side with
/// NULL, a union adds the rows of its other inputs, and a grouping set fills
diff --git a/datafusion/sqllogictest/test_files/cte_materialized.slt b/datafusion/sqllogictest/test_files/cte_materialized.slt
index 663263a720..094f30ee02 100644
--- a/datafusion/sqllogictest/test_files/cte_materialized.slt
+++ b/datafusion/sqllogictest/test_files/cte_materialized.slt
@@ -210,6 +210,22 @@ SELECT k FROM t WHERE v IN (
3
4
+# A LIMIT at the root of the continuation does not change whether it is empty.
+query I rowsort
+SELECT k FROM t WHERE EXISTS (
+ WITH r AS MATERIALIZED (SELECT k AS rk FROM t)
+ SELECT 1 FROM r WHERE r.rk = t.k + 1 LIMIT 1)
+----
+1
+2
+3
+
+# A filter above the LIMIT reads the row the LIMIT kept, so it stays correlated.
+query error Physical plan does not support logical expression Exists
+SELECT k FROM t WHERE EXISTS (
+ WITH r AS MATERIALIZED (SELECT k AS rk, v AS rv FROM t)
+ SELECT 1 FROM (SELECT * FROM r WHERE r.rk = t.k LIMIT 1) s WHERE s.rv > 0)
+
# A body that refers to the outer query would have to run once per outer row.
# Such a CTE is inlined, as if MATERIALIZED was not given.
query I rowsort
diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt
index 999f415c3c..2ec240ac67 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -779,6 +779,75 @@ SELECT t1_id FROM t1 WHERE EXISTS (SELECT 1 FROM (SELECT * FROM t2 WHERE t2_id =
----
22
+#exists_subquery_with_limit_below_having
+#not de-correlated, the limit decides the count the HAVING reads: count(*) over
+#the one row LIMIT 1 keeps can never exceed 1, so the subquery is always empty.
+#Dropping the limit counted the whole scan and this returned 33.
+#https://github.com/apache/datafusion/issues/26629
+query TT
+explain SELECT t1_id FROM t1 WHERE EXISTS (SELECT 1 FROM (SELECT t2_id FROM t2 WHERE t2_int = t1_int LIMIT 1) s HAVING count(*) > 1)
+----
+logical_plan
+01)Projection: t1.t1_id
+02)--Filter: EXISTS (<subquery>)
+03)----Subquery:
+04)------Projection: Int64(1)
+05)--------Filter: count(Int64(1)) > Int64(1)
+06)----------Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]
+07)------------SubqueryAlias: s
+08)--------------Projection: t2.t2_id
+09)----------------Limit: skip=0, fetch=1
+10)------------------Filter: t2.t2_int = outer_ref(t1.t1_int)
+11)--------------------TableScan: t2
+12)----TableScan: t1 projection=[t1_id, t1_int]
+
+# errors rather than returning wrong rows
+query error Physical plan does not support logical expression Exists
+SELECT t1_id FROM t1 WHERE EXISTS (SELECT 1 FROM (SELECT t2_id FROM t2 WHERE t2_int = t1_int LIMIT 1) s HAVING count(*) > 1)
+
+# same for NOT EXISTS, which returned 11, 22, 44 instead of every row
+query error Physical plan does not support logical expression Exists
+SELECT t1_id FROM t1 WHERE NOT EXISTS (SELECT 1 FROM (SELECT t2_id FROM t2 WHERE t2_int = t1_int LIMIT 1) s HAVING count(*) > 1)
+
+#exists_subquery_with_limit_below_filter
+#not de-correlated, the rows the limit keeps decide the filter above it
+query error Physical plan does not support logical expression Exists
+SELECT t1_id FROM t1 WHERE EXISTS (SELECT 1 FROM (SELECT t2_id, t2_int FROM t2 WHERE t2_id = t1_id LIMIT 1) s WHERE s.t2_int > 2)
+
+#exists_subquery_with_limit_below_aggregate
+#de-correlated, an aggregate without a grouping always returns its one row, so
+#the rows the limit held back cannot change whether the subquery is empty
+query TT
+explain SELECT t1_id FROM t1 WHERE EXISTS (SELECT count(*) FROM (SELECT t2_id FROM t2 WHERE t2_id = t1_id LIMIT 1) s)
+----
+logical_plan TableScan: t1 projection=[t1_id]
+
+query I rowsort
+SELECT t1_id FROM t1 WHERE EXISTS (SELECT count(*) FROM (SELECT t2_id FROM t2 WHERE t2_id = t1_id LIMIT 1) s)
+----
+11
+22
+33
+44
+
+#exists_subquery_with_limit_below_distinct
+#de-correlated, a DISTINCT has rows exactly when its input has rows
+query I rowsort
+SELECT t1_id FROM t1 WHERE EXISTS (SELECT DISTINCT s.t2_id FROM (SELECT t2_id FROM t2 WHERE t2_id = t1_id LIMIT 1) s)
+----
+11
+22
+44
+
+#exists_subquery_with_limit_below_group_by
+#de-correlated, a grouping has rows exactly when its input has rows
+query I rowsort
+SELECT t1_id FROM t1 WHERE EXISTS (SELECT s.t2_id FROM (SELECT t2_id FROM t2 WHERE t2_id = t1_id LIMIT 1) s GROUP BY s.t2_id)
+----
+11
+22
+44
+
#in_correlated_subquery_with_limit
#not de-correlated
query TT