Skip to content

Apache DataFusion SQL Query Engine

fix: error instead of wrong rows for a LATERAL with a HAVING over an ungrouped aggregate (#26874)

RAGHVENDRA KUMAR YADAVa16d10da9ed2
diff --git a/datafusion/optimizer/src/decorrelate_lateral_join.rs b/datafusion/optimizer/src/decorrelate_lateral_join.rs
index 085d7bd9d3..70c36a6610 100644
--- a/datafusion/optimizer/src/decorrelate_lateral_join.rs
+++ b/datafusion/optimizer/src/decorrelate_lateral_join.rs
@@ -19,7 +19,7 @@ use datafusion_common::tree_node::{
use datafusion_common::{Column, DFSchema, Result, ScalarValue, TableReference};
use datafusion_expr::logical_plan::{JoinType, Subquery};
use datafusion_expr::utils::conjunction;
-use datafusion_expr::{LogicalPlan, LogicalPlanBuilder, SubqueryAlias};
+use datafusion_expr::{Filter, LogicalPlan, LogicalPlanBuilder, SubqueryAlias};
/// Optimizer rule for rewriting lateral joins to joins
#[derive(Default, Debug)]
@@ -97,6 +97,22 @@ fn rewrite_internal(join: Join) -> Result<Transformed<LogicalPlan>> {
return Ok(Transformed::no(LogicalPlan::Join(join)));
}
+ // A HAVING over an ungrouped aggregate can remove the aggregate's only
+ // row. The rewrite below handles neither case correctly, so leave the join
+ // correlated and let the query error instead of returning wrong rows.
+ // <https://github.com/apache/datafusion/issues/21198>
+ // - An inner lateral join becomes an outer join that keeps every outer row,
+ // so an outer row whose row the HAVING removes comes back with NULLs.
+ // - A HAVING that reads an outer column is moved into the join condition,
+ // where the count bug handling turns its unmatched rows into a count of 0,
+ // for a left lateral join too.
+ if pull_up.pulled_up_scalar_agg
+ && let Some(having) = having_over_ungrouped_aggregate(subquery_plan)
+ && (original_join_type == JoinType::Inner || having.predicate.contains_outer())
+ {
+ return Ok(Transformed::no(LogicalPlan::Join(join)));
+ }
+
// TODO: support HAVING in lateral subqueries.
// <https://github.com/apache/datafusion/issues/21198>
if pull_up.pull_up_having_expr.is_some() {
@@ -315,6 +331,39 @@ fn rewrite_internal(join: Join) -> Result<Transformed<LogicalPlan>> {
Ok(Transformed::new(new_plan, true, TreeNodeRecursion::Jump))
}
+/// The `HAVING` of a subquery whose result is an ungrouped aggregate: a
+/// `Filter` between the subquery's output and that aggregate, with only
+/// projections and aliases around them. A filter below the aggregate only
+/// changes the aggregate's input, so it is not returned.
+fn having_over_ungrouped_aggregate(plan: &LogicalPlan) -> Option<&Filter> {
+ match plan {
+ LogicalPlan::Projection(projection) => {
+ having_over_ungrouped_aggregate(&projection.input)
+ }
+ LogicalPlan::SubqueryAlias(alias) => {
+ having_over_ungrouped_aggregate(&alias.input)
+ }
+ LogicalPlan::Filter(filter) => {
+ feeds_ungrouped_aggregate(&filter.input).then_some(filter)
+ }
+ _ => None,
+ }
+}
+
+/// Whether `plan` is an ungrouped aggregate, seen through projections,
+/// aliases and filters.
+fn feeds_ungrouped_aggregate(plan: &LogicalPlan) -> bool {
+ match plan {
+ LogicalPlan::Aggregate(aggregate) => aggregate.group_expr.is_empty(),
+ LogicalPlan::Projection(projection) => {
+ feeds_ungrouped_aggregate(&projection.input)
+ }
+ LogicalPlan::SubqueryAlias(alias) => feeds_ungrouped_aggregate(&alias.input),
+ LogicalPlan::Filter(filter) => feeds_ungrouped_aggregate(&filter.input),
+ _ => false,
+ }
+}
+
/// Extract the Subquery and optional alias from a lateral join's right side.
fn extract_lateral_subquery(
plan: &LogicalPlan,
diff --git a/datafusion/sqllogictest/test_files/lateral_join.slt b/datafusion/sqllogictest/test_files/lateral_join.slt
index e442403728..ef76f47f9e 100644
--- a/datafusion/sqllogictest/test_files/lateral_join.slt
+++ b/datafusion/sqllogictest/test_files/lateral_join.slt
@@ -726,6 +726,92 @@ FROM t1, LATERAL (
) AS sub
ORDER BY t1.id;
+# HAVING count(*) > 1 is false on an empty group, so it is not pulled up and
+# stays above the aggregate. The inner lateral join must drop t1.id=2 (cnt=1)
+# and t1.id=3 (cnt=0), so until that is supported the query errors instead
+# of returning them with a NULL cnt.
+# Expected result once supported: only t1.id=1 (cnt=2).
+statement error OuterReferenceColumn
+SELECT t1.id, sub.cnt
+FROM t1, LATERAL (
+ SELECT count(*) AS cnt FROM t2 WHERE t2.t1_id = t1.id HAVING count(*) > 1
+) AS sub
+ORDER BY t1.id;
+
+# The same with a HAVING on another aggregate than the one returned.
+statement error OuterReferenceColumn
+SELECT t1.id, sub.cnt
+FROM t1, LATERAL (
+ SELECT count(*) AS cnt FROM t2 WHERE t2.t1_id = t1.id HAVING max(t2.id) > 15
+) AS sub
+ORDER BY t1.id;
+
+# A left lateral join keeps the outer rows the HAVING removes, with NULLs, so
+# it is still decorrelated when the HAVING does not read an outer column.
+query II
+SELECT t1.id, sub.cnt
+FROM t1 LEFT JOIN LATERAL (
+ SELECT count(*) AS cnt FROM t2 WHERE t2.t1_id = t1.id HAVING count(*) > 1
+) AS sub ON true
+ORDER BY t1.id;
+----
+1 2
+2 NULL
+3 NULL
+
+# A HAVING that reads an outer column is moved into the join condition, where
+# rows it rejects became a count of 0, for an inner and a left lateral join.
+# Expected result once supported: only t1.id=1 (cnt=2) for the inner join, and
+# (1, 2), (2, NULL), (3, NULL) for the left join.
+statement error OuterReferenceColumn
+SELECT t1.id, sub.cnt
+FROM t1, LATERAL (
+ SELECT count(*) AS cnt FROM t2 WHERE t2.t1_id = t1.id HAVING count(*) > t1.id
+) AS sub
+ORDER BY t1.id;
+
+statement error OuterReferenceColumn
+SELECT t1.id, sub.cnt
+FROM t1 LEFT JOIN LATERAL (
+ SELECT count(*) AS cnt FROM t2 WHERE t2.t1_id = t1.id HAVING count(*) > t1.id
+) AS sub ON true
+ORDER BY t1.id;
+
+# Without a correlated WHERE the aggregate keeps its single row for every
+# outer row, so a HAVING that reads an outer column is a plain join filter.
+query II
+SELECT t1.id, sub.cnt
+FROM t1, LATERAL (
+ SELECT count(*) AS cnt FROM t2 HAVING count(*) > t1.id
+) AS sub
+ORDER BY t1.id;
+----
+1 3
+2 3
+
+query II
+SELECT t1.id, sub.cnt
+FROM t1 LEFT JOIN LATERAL (
+ SELECT count(*) AS cnt FROM t2 HAVING count(*) > t1.id
+) AS sub ON true
+ORDER BY t1.id;
+----
+1 3
+2 3
+3 NULL
+
+# With GROUP BY, a group the HAVING removes has no row at all, so an inner
+# lateral join is still decorrelated.
+query III
+SELECT t1.id, sub.t1_id, sub.cnt
+FROM t1, LATERAL (
+ SELECT t2.t1_id, count(*) AS cnt FROM t2 WHERE t2.t1_id = t1.id
+ GROUP BY t2.t1_id HAVING count(*) > 1
+) AS sub
+ORDER BY t1.id;
+----
+1 1 2
+
# --- Correlated filter on the nullable side of an outer join ---
# https://github.com/apache/datafusion/issues/25507
# Pulling `t2.t1_id = t1.id` above the LEFT JOIN would drop the t3 rows that