Apache DataFusion SQL Query Engine
fix: error instead of wrong rows for a LATERAL with a HAVING over an ungrouped aggregate (#26874)
RAGHVENDRA KUMAR YADAVa16d10da9ed2
diff --git a/datafusion/optimizer/src/decorrelate_lateral_join.rs b/datafusion/optimizer/src/decorrelate_lateral_join.rsindex 085d7bd9d3..70c36a6610 100644--- a/datafusion/optimizer/src/decorrelate_lateral_join.rs+++ b/datafusion/optimizer/src/decorrelate_lateral_join.rs@@ -19,7 +19,7 @@ use datafusion_common::tree_node::{use datafusion_common::{Column, DFSchema, Result, ScalarValue, TableReference};use datafusion_expr::logical_plan::{JoinType, Subquery};use datafusion_expr::utils::conjunction;-use datafusion_expr::{LogicalPlan, LogicalPlanBuilder, SubqueryAlias};+use datafusion_expr::{Filter, LogicalPlan, LogicalPlanBuilder, SubqueryAlias};/// Optimizer rule for rewriting lateral joins to joins#[derive(Default, Debug)]@@ -97,6 +97,22 @@ fn rewrite_internal(join: Join) -> Result<Transformed<LogicalPlan>> {return Ok(Transformed::no(LogicalPlan::Join(join)));}+ // A HAVING over an ungrouped aggregate can remove the aggregate's only+ // row. The rewrite below handles neither case correctly, so leave the join+ // correlated and let the query error instead of returning wrong rows.+ // <https://github.com/apache/datafusion/issues/21198>+ // - An inner lateral join becomes an outer join that keeps every outer row,+ // so an outer row whose row the HAVING removes comes back with NULLs.+ // - A HAVING that reads an outer column is moved into the join condition,+ // where the count bug handling turns its unmatched rows into a count of 0,+ // for a left lateral join too.+ if pull_up.pulled_up_scalar_agg+ && let Some(having) = having_over_ungrouped_aggregate(subquery_plan)+ && (original_join_type == JoinType::Inner || having.predicate.contains_outer())+ {+ return Ok(Transformed::no(LogicalPlan::Join(join)));+ }+// TODO: support HAVING in lateral subqueries.// <https://github.com/apache/datafusion/issues/21198>if pull_up.pull_up_having_expr.is_some() {@@ -315,6 +331,39 @@ fn rewrite_internal(join: Join) -> Result<Transformed<LogicalPlan>> {Ok(Transformed::new(new_plan, true, TreeNodeRecursion::Jump))}+/// The `HAVING` of a subquery whose result is an ungrouped aggregate: a+/// `Filter` between the subquery's output and that aggregate, with only+/// projections and aliases around them. A filter below the aggregate only+/// changes the aggregate's input, so it is not returned.+fn having_over_ungrouped_aggregate(plan: &LogicalPlan) -> Option<&Filter> {+ match plan {+ LogicalPlan::Projection(projection) => {+ having_over_ungrouped_aggregate(&projection.input)+ }+ LogicalPlan::SubqueryAlias(alias) => {+ having_over_ungrouped_aggregate(&alias.input)+ }+ LogicalPlan::Filter(filter) => {+ feeds_ungrouped_aggregate(&filter.input).then_some(filter)+ }+ _ => None,+ }+}++/// Whether `plan` is an ungrouped aggregate, seen through projections,+/// aliases and filters.+fn feeds_ungrouped_aggregate(plan: &LogicalPlan) -> bool {+ match plan {+ LogicalPlan::Aggregate(aggregate) => aggregate.group_expr.is_empty(),+ LogicalPlan::Projection(projection) => {+ feeds_ungrouped_aggregate(&projection.input)+ }+ LogicalPlan::SubqueryAlias(alias) => feeds_ungrouped_aggregate(&alias.input),+ LogicalPlan::Filter(filter) => feeds_ungrouped_aggregate(&filter.input),+ _ => false,+ }+}+/// Extract the Subquery and optional alias from a lateral join's right side.fn extract_lateral_subquery(plan: &LogicalPlan,diff --git a/datafusion/sqllogictest/test_files/lateral_join.slt b/datafusion/sqllogictest/test_files/lateral_join.sltindex e442403728..ef76f47f9e 100644--- a/datafusion/sqllogictest/test_files/lateral_join.slt+++ b/datafusion/sqllogictest/test_files/lateral_join.slt@@ -726,6 +726,92 @@ FROM t1, LATERAL () AS subORDER BY t1.id;+# HAVING count(*) > 1 is false on an empty group, so it is not pulled up and+# stays above the aggregate. The inner lateral join must drop t1.id=2 (cnt=1)+# and t1.id=3 (cnt=0), so until that is supported the query errors instead+# of returning them with a NULL cnt.+# Expected result once supported: only t1.id=1 (cnt=2).+statement error OuterReferenceColumn+SELECT t1.id, sub.cnt+FROM t1, LATERAL (+ SELECT count(*) AS cnt FROM t2 WHERE t2.t1_id = t1.id HAVING count(*) > 1+) AS sub+ORDER BY t1.id;++# The same with a HAVING on another aggregate than the one returned.+statement error OuterReferenceColumn+SELECT t1.id, sub.cnt+FROM t1, LATERAL (+ SELECT count(*) AS cnt FROM t2 WHERE t2.t1_id = t1.id HAVING max(t2.id) > 15+) AS sub+ORDER BY t1.id;++# A left lateral join keeps the outer rows the HAVING removes, with NULLs, so+# it is still decorrelated when the HAVING does not read an outer column.+query II+SELECT t1.id, sub.cnt+FROM t1 LEFT JOIN LATERAL (+ SELECT count(*) AS cnt FROM t2 WHERE t2.t1_id = t1.id HAVING count(*) > 1+) AS sub ON true+ORDER BY t1.id;+----+1 2+2 NULL+3 NULL++# A HAVING that reads an outer column is moved into the join condition, where+# rows it rejects became a count of 0, for an inner and a left lateral join.+# Expected result once supported: only t1.id=1 (cnt=2) for the inner join, and+# (1, 2), (2, NULL), (3, NULL) for the left join.+statement error OuterReferenceColumn+SELECT t1.id, sub.cnt+FROM t1, LATERAL (+ SELECT count(*) AS cnt FROM t2 WHERE t2.t1_id = t1.id HAVING count(*) > t1.id+) AS sub+ORDER BY t1.id;++statement error OuterReferenceColumn+SELECT t1.id, sub.cnt+FROM t1 LEFT JOIN LATERAL (+ SELECT count(*) AS cnt FROM t2 WHERE t2.t1_id = t1.id HAVING count(*) > t1.id+) AS sub ON true+ORDER BY t1.id;++# Without a correlated WHERE the aggregate keeps its single row for every+# outer row, so a HAVING that reads an outer column is a plain join filter.+query II+SELECT t1.id, sub.cnt+FROM t1, LATERAL (+ SELECT count(*) AS cnt FROM t2 HAVING count(*) > t1.id+) AS sub+ORDER BY t1.id;+----+1 3+2 3++query II+SELECT t1.id, sub.cnt+FROM t1 LEFT JOIN LATERAL (+ SELECT count(*) AS cnt FROM t2 HAVING count(*) > t1.id+) AS sub ON true+ORDER BY t1.id;+----+1 3+2 3+3 NULL++# With GROUP BY, a group the HAVING removes has no row at all, so an inner+# lateral join is still decorrelated.+query III+SELECT t1.id, sub.t1_id, sub.cnt+FROM t1, LATERAL (+ SELECT t2.t1_id, count(*) AS cnt FROM t2 WHERE t2.t1_id = t1.id+ GROUP BY t2.t1_id HAVING count(*) > 1+) AS sub+ORDER BY t1.id;+----+1 1 2+# --- Correlated filter on the nullable side of an outer join ---# https://github.com/apache/datafusion/issues/25507# Pulling `t2.t1_id = t1.id` above the LEFT JOIN would drop the t3 rows that