Apache DataFusion SQL Query Engine
fix: keep a LIMIT inside EXISTS that an operator above it can empty (#26640)
RAGHVENDRA KUMAR YADAV1aa6c294bd01
diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rsindex e1c6235f31..5143ac87c1 100644--- a/datafusion/optimizer/src/decorrelate.rs+++ b/datafusion/optimizer/src/decorrelate.rs@@ -97,6 +97,12 @@ pub struct PullUpCorrelatedExpr {/// grouping set that omits a correlated column can tell whether/// something above it reads the column the pull up would fill in.column_refs_above: Vec<BTreeSet<Column>>,+ /// For each node on that same path, whether it returns rows based only on+ /// whether its input has rows (see [`depends_only_on_input_having_rows`]). One+ /// entry per node, pushed and popped with [`Self::column_refs_above`].+ /// `f_down`'s `Limit` arm reads it: a `Limit` is only dropped from an+ /// `EXISTS` subquery when this holds for every node above it.+ depends_only_on_input_having_rows_above: Vec<bool>,}impl Default for PullUpCorrelatedExpr {@@ -121,6 +127,7 @@ impl PullUpCorrelatedExpr {pulled_up_scalar_agg: false,correlated_filters: Vec::new(),column_refs_above: Vec::new(),+ depends_only_on_input_having_rows_above: Vec::new(),}}@@ -183,6 +190,8 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {column_refs.extend(expr.column_refs().into_iter().cloned());}self.column_refs_above.push(column_refs);+ self.depends_only_on_input_having_rows_above+ .push(depends_only_on_input_having_rows(&plan));// This match is exhaustive on purpose. A correlated filter moves above// a node only when the node is known to keep the result the same. Add@@ -268,9 +277,8 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {if !self.exists_sub_query {return Ok(self.stop_pull_up(plan));}- // Only emptiness matters for EXISTS, so remove a limit that- // cannot make its input empty and replace one that always does- // with an empty relation.+ // A `LIMIT 0` always gives an empty input, whatever is+ // above it, so it becomes an empty relation.let fetch = limit.get_fetch_type()?;if matches!(fetch, FetchType::Literal(Some(0))) {return Ok(Transformed::yes(LogicalPlan::EmptyRelation(@@ -280,6 +288,20 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {},)));}+ // `EXISTS` asks whether the whole subquery returns a row, not+ // whether this `Limit` does. If a node above the `Limit` looks+ // at which rows it gets, such as a `Filter` or the `HAVING` in+ // https://github.com/apache/datafusion/issues/26629, removing+ // the `Limit` hands it different rows and can change the+ // answer, so the `Limit` has to stay.+ let (_, above) = self+ .depends_only_on_input_having_rows_above+ .split_last()+ .expect("f_down pushed this Limit's own entry");+ if !above.iter().all(|follows| *follows) {+ return Ok(self.stop_pull_up(plan));+ }+match (limit.get_skip_type()?, fetch) {(SkipType::Literal(0), FetchType::Literal(_)) => {// The rewriter does not call `f_down` on the returned@@ -289,6 +311,7 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {// `f_up` call reaches this tree position once `Limit`// is gone, so only one entry must remain pushed.self.column_refs_above.pop();+ self.depends_only_on_input_having_rows_above.pop();let mut t = self.f_down((*limit.input).clone())?;t.transformed = true;Ok(t)@@ -364,6 +387,7 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {// Pop this node's own entry, pushed by the matching `f_down` call, so// `column_refs_above` holds only the nodes strictly above `plan`.self.column_refs_above.pop();+ self.depends_only_on_input_having_rows_above.pop();// A node that null-extends or regroups rows can put a NULL back into// a column that a correlated filter below it rejected, so the filters@@ -897,6 +921,43 @@ impl PullUpCorrelatedExpr {}}+/// Whether `plan` returns rows based only on whether its input has rows, and+/// not on which rows those are.+///+/// This is what lets `EXISTS` drop a `Limit` with no skip and a non-zero+/// fetch. Such a `Limit` returns rows exactly when its input has rows. If+/// every node above it also only depends on whether it gets rows, the whole+/// subquery returns rows exactly when the `Limit`'s input has rows, with or+/// without the `Limit`, so removing it cannot change the `EXISTS` result.+///+/// It holds for:+/// - `Projection`, `SubqueryAlias`, `Sort`, `Window` and `Repartition`, which+/// return rows whenever their input has rows (a `Sort` with a fetch of 0+/// returns none either way)+/// - `DISTINCT` and a grouped `Aggregate`, which return at least one row for a+/// non-empty input and none for an empty one+/// - an `Aggregate` without grouping, which always returns one row+/// - a materialized CTE, which returns its continuation's rows+///+/// It does not hold for a `Filter`, which keeps or removes rows by their+/// values, so the rows the `Limit` kept decide whether any survive. A `Join`+/// and an `Unnest` can also turn rows into none. Every other plan, including a+/// user-defined `Extension`, is treated as not holding: the `Limit` stays and+/// the subquery is not decorrelated.+fn depends_only_on_input_having_rows(plan: &LogicalPlan) -> bool {+ matches!(+ plan,+ LogicalPlan::Projection(_)+ | LogicalPlan::SubqueryAlias(_)+ | LogicalPlan::Sort(_)+ | LogicalPlan::Window(_)+ | LogicalPlan::Repartition(_)+ | LogicalPlan::Distinct(Distinct::All(_))+ | LogicalPlan::Aggregate(_)+ | LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(_))+ )+}+/// Whether `plan` can output a NULL in a column that a filter below it keeps/// free of NULLs: an outer join fills the columns of its unmatched side with/// NULL, a union adds the rows of its other inputs, and a grouping set fillsdiff --git a/datafusion/sqllogictest/test_files/cte_materialized.slt b/datafusion/sqllogictest/test_files/cte_materialized.sltindex 663263a720..094f30ee02 100644--- a/datafusion/sqllogictest/test_files/cte_materialized.slt+++ b/datafusion/sqllogictest/test_files/cte_materialized.slt@@ -210,6 +210,22 @@ SELECT k FROM t WHERE v IN (34+# A LIMIT at the root of the continuation does not change whether it is empty.+query I rowsort+SELECT k FROM t WHERE EXISTS (+ WITH r AS MATERIALIZED (SELECT k AS rk FROM t)+ SELECT 1 FROM r WHERE r.rk = t.k + 1 LIMIT 1)+----+1+2+3++# A filter above the LIMIT reads the row the LIMIT kept, so it stays correlated.+query error Physical plan does not support logical expression Exists+SELECT k FROM t WHERE EXISTS (+ WITH r AS MATERIALIZED (SELECT k AS rk, v AS rv FROM t)+ SELECT 1 FROM (SELECT * FROM r WHERE r.rk = t.k LIMIT 1) s WHERE s.rv > 0)+# A body that refers to the outer query would have to run once per outer row.# Such a CTE is inlined, as if MATERIALIZED was not given.query I rowsortdiff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex 999f415c3c..2ec240ac67 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -779,6 +779,75 @@ SELECT t1_id FROM t1 WHERE EXISTS (SELECT 1 FROM (SELECT * FROM t2 WHERE t2_id =----22+#exists_subquery_with_limit_below_having+#not de-correlated, the limit decides the count the HAVING reads: count(*) over+#the one row LIMIT 1 keeps can never exceed 1, so the subquery is always empty.+#Dropping the limit counted the whole scan and this returned 33.+#https://github.com/apache/datafusion/issues/26629+query TT+explain SELECT t1_id FROM t1 WHERE EXISTS (SELECT 1 FROM (SELECT t2_id FROM t2 WHERE t2_int = t1_int LIMIT 1) s HAVING count(*) > 1)+----+logical_plan+01)Projection: t1.t1_id+02)--Filter: EXISTS (<subquery>)+03)----Subquery:+04)------Projection: Int64(1)+05)--------Filter: count(Int64(1)) > Int64(1)+06)----------Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]+07)------------SubqueryAlias: s+08)--------------Projection: t2.t2_id+09)----------------Limit: skip=0, fetch=1+10)------------------Filter: t2.t2_int = outer_ref(t1.t1_int)+11)--------------------TableScan: t2+12)----TableScan: t1 projection=[t1_id, t1_int]++# errors rather than returning wrong rows+query error Physical plan does not support logical expression Exists+SELECT t1_id FROM t1 WHERE EXISTS (SELECT 1 FROM (SELECT t2_id FROM t2 WHERE t2_int = t1_int LIMIT 1) s HAVING count(*) > 1)++# same for NOT EXISTS, which returned 11, 22, 44 instead of every row+query error Physical plan does not support logical expression Exists+SELECT t1_id FROM t1 WHERE NOT EXISTS (SELECT 1 FROM (SELECT t2_id FROM t2 WHERE t2_int = t1_int LIMIT 1) s HAVING count(*) > 1)++#exists_subquery_with_limit_below_filter+#not de-correlated, the rows the limit keeps decide the filter above it+query error Physical plan does not support logical expression Exists+SELECT t1_id FROM t1 WHERE EXISTS (SELECT 1 FROM (SELECT t2_id, t2_int FROM t2 WHERE t2_id = t1_id LIMIT 1) s WHERE s.t2_int > 2)++#exists_subquery_with_limit_below_aggregate+#de-correlated, an aggregate without a grouping always returns its one row, so+#the rows the limit held back cannot change whether the subquery is empty+query TT+explain SELECT t1_id FROM t1 WHERE EXISTS (SELECT count(*) FROM (SELECT t2_id FROM t2 WHERE t2_id = t1_id LIMIT 1) s)+----+logical_plan TableScan: t1 projection=[t1_id]++query I rowsort+SELECT t1_id FROM t1 WHERE EXISTS (SELECT count(*) FROM (SELECT t2_id FROM t2 WHERE t2_id = t1_id LIMIT 1) s)+----+11+22+33+44++#exists_subquery_with_limit_below_distinct+#de-correlated, a DISTINCT has rows exactly when its input has rows+query I rowsort+SELECT t1_id FROM t1 WHERE EXISTS (SELECT DISTINCT s.t2_id FROM (SELECT t2_id FROM t2 WHERE t2_id = t1_id LIMIT 1) s)+----+11+22+44++#exists_subquery_with_limit_below_group_by+#de-correlated, a grouping has rows exactly when its input has rows+query I rowsort+SELECT t1_id FROM t1 WHERE EXISTS (SELECT s.t2_id FROM (SELECT t2_id FROM t2 WHERE t2_id = t1_id LIMIT 1) s GROUP BY s.t2_id)+----+11+22+44+#in_correlated_subquery_with_limit#not de-correlatedquery TT