Apache DataFusion SQL Query Engine
fix: return a query error for correlated scalar subquery with outer-column HAVING over count aggregates (#26723)
siddu9d15cf079fe9
diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rsindex 5143ac87c1..80500b89bf 100644--- a/datafusion/optimizer/src/decorrelate.rs+++ b/datafusion/optimizer/src/decorrelate.rs@@ -68,6 +68,14 @@ pub struct PullUpCorrelatedExpr {pub collected_count_expr_map: HashMap<LogicalPlan, ExprResultMap>,/// pull up having expr, which must be evaluated after the Joinpub pull_up_having_expr: Option<Expr>,+ /// Outer-reference conditions from a `Filter` that sits directly above a+ /// count-bug `Aggregate`: a `HAVING` clause (or an outer-only filter)+ /// that filters the aggregate *output*. Unlike `join_filters`, these do+ /// not merely correlate the two sides: an unmatched LEFT JOIN row may+ /// mean one of them rejected the group rather than that no group+ /// matched, and a scalar subquery must yield NULL for a rejected group.+ /// `build_join` uses them to reject such queries up front.+ pub having_like_join_filters: Vec<Expr>,/// whether we have converted a scalar aggregation into a group aggregation. When unnesting/// lateral joins, we need to produce a left outer join in such cases.///@@ -124,6 +132,7 @@ impl PullUpCorrelatedExpr {need_handle_count_bug: false,collected_count_expr_map: HashMap::new(),pull_up_having_expr: None,+ having_like_join_filters: Vec::new(),pulled_up_scalar_agg: false,correlated_filters: Vec::new(),column_refs_above: Vec::new(),@@ -415,6 +424,22 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {// in_predicate may be already included in the join filters, remove it from the join filters first.join_filters = remove_duplicated_filter(join_filters, in_predicate)?;}+ // A `Filter` directly above a count-bug `Aggregate` is a+ // HAVING clause (or an outer-only filter above the aggregate):+ // its outer-reference conditions filter the aggregate *output*,+ // so a LEFT JOIN row can be unmatched either because the+ // correlation matched no group or because one of them rejected+ // the group. Record them so the scalar rewrite can tell the+ // two apart; where the output needs count-bug compensation it+ // rejects the query instead of guessing. A `Filter` below the+ // aggregate only prunes input rows and is left out.+ if self+ .collected_count_expr_map+ .contains_key(&*plan_filter.input)+ {+ self.having_like_join_filters+ .extend(join_filters.iter().cloned());+ }let correlated_subquery_cols =collect_subquery_cols(&join_filters, subquery_schema)?;for expr in join_filters {diff --git a/datafusion/optimizer/src/scalar_subquery_to_join.rs b/datafusion/optimizer/src/scalar_subquery_to_join.rsindex 126b97aec3..7dac77768c 100644--- a/datafusion/optimizer/src/scalar_subquery_to_join.rs+++ b/datafusion/optimizer/src/scalar_subquery_to_join.rs@@ -349,6 +349,35 @@ fn build_join(.alias(subquery_alias.to_string())?.build()?;+ // A HAVING or outer-only filter recorded from above the aggregate cannot+ // be rewritten into the join when the output column gets a compensation+ // CASE below. Kept in the ON, a row the condition rejects looks unmatched+ // and falls back to the empty-input default, which is the issue's wrong 0. Checked+ // in the compensation CASE instead, the condition would have to be+ // evaluated twice, against the defaults for an unmatched row and against+ // the actual output for a matched one, and neither evaluation is right+ // for the rows the other one covers. The gate looks at the output column+ // alone because the map also holds aggregates that appear only in the+ // HAVING, whose NULL defaults need no CASE; an output whose own default+ // is NULL (the subquery projects `max`, say) keeps filtering in the ON as+ // before, which is correct there because an unmatched row is NULL either+ // way. Rather than guess, reject the query.+ let output_name = subquery_plan.schema().field(0).name();+ let output_needs_compensation = match collected_count_expr_map+ .as_ref()+ .and_then(|expr_map| expr_map.get(output_name))+ {+ Some(default_expr) => {+ !evaluates_to_null(default_expr.clone(), default_expr.column_refs())?+ }+ None => false,+ };+ if output_needs_compensation && !pull_up.having_like_join_filters.is_empty() {+ return plan_err!(+ "Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns, and this rewrite cannot evaluate it against both the group and the empty input without risking wrong results. Move the condition outside the subquery"+ );+ }+let join_filter_opt = build_join_filter(pull_up.join_filters,pull_up.correlated_subquery_cols_map.values().flatten(),diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex 4b553cc9aa..8a6143bc68 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -2219,6 +2219,71 @@ WHERE t1_int + 2 NOT IN (SELECT max(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id)query error Physical plan does not support logical expression InSubquerySELECT t1_id, t1_int + 2 IN (SELECT max(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id) FROM t1+# https://github.com/apache/datafusion/issues/26503+# A correlated scalar subquery whose HAVING reads an outer column is rejected:+# left in the join ON it returns the empty-input 0 where NULL is correct.+statement error .*Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns+SELECT x.a, x.d,+ (SELECT count(*) FROM (VALUES (1)) AS y(c) WHERE y.c = x.d HAVING count(*) >= x.a) AS v+FROM (VALUES (1, 1), (2, 1), (1, 9)) AS x(a, d)+ORDER BY x.d, x.a++# Same shape with the predicate in a filter above the aggregate rather than in+# HAVING: the filter references only an outer column, so it is rejected too.+statement error .*Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns+SELECT (SELECT n FROM (SELECT count(*) AS n FROM (VALUES (1)) AS y(c) WHERE y.c = x.d) WHERE x.a > 1) AS v+FROM (VALUES (1, 1)) AS x(a, d)++# A local HAVING that holds on the empty-input default still yields the+# default (0), not NULL, when no inner row matches the correlation. Only+# conditions that read outer columns are rejected.+query I+SELECT (SELECT count(*) FROM (VALUES (1)) AS y(c) WHERE y.c = x.d HAVING count(*) >= 0) AS v+FROM (VALUES (1, 9)) AS x(a, d)+----+0++# A local HAVING that fails on the empty-input default yields NULL.+query I+SELECT (SELECT count(*) FROM (VALUES (1)) AS y(c) WHERE y.c = x.d HAVING count(*) > 0) AS v+FROM (VALUES (1, 9)) AS x(a, d)+----+NULL++# A stacked filter above the aggregate that reads an outer column is rejected+# for the same reason, whether or not a HAVING keyword is written.+statement error .*Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns+SELECT x.n, x.d,+ (SELECT s.n FROM (SELECT count(*) AS n FROM (VALUES (1)) AS y(c) WHERE y.c = x.d) AS s+ WHERE s.n < x.n) AS v+FROM (VALUES (5, 9), (0, 9), (5, 1)) AS x(n, d)+ORDER BY x.d, x.n++# A group the HAVING rejects would fall back to the empty-input default when+# the condition stays in the ON, so this shape is rejected as well.+statement error .*Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns+SELECT x.a, x.d,+ (SELECT count(*) FROM (VALUES (1)) AS y(c) WHERE y.c = x.d HAVING count(*) < x.a) AS v+FROM (VALUES (1, 1), (2, 1), (1, 9)) AS x(a, d)+ORDER BY x.d, x.a++# max appears only in the HAVING, but the subquery outputs count(*), so the+# output still needs compensation and the query is rejected.+statement error .*Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns+SELECT x.a, x.d,+ (SELECT count(*) FROM (VALUES (1)) AS y(c) WHERE y.c = x.d HAVING max(y.c) < x.a) AS v+FROM (VALUES (1, 1), (2, 1), (1, 9)) AS x(a, d)+ORDER BY x.d, x.a++# The same shape, but here the HAVING does hold on the empty input+# (max(y.c) IS NULL). The join cannot tell which side decides the unmatched+# row, so this one is rejected too.+statement error .*Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns+SELECT x.a, x.d,+ (SELECT count(*) FROM (VALUES (1)) AS y(c) WHERE y.c = x.d HAVING max(y.c) IS NULL OR count(*) < x.a) AS v+FROM (VALUES (1, 1), (2, 1), (1, 9)) AS x(a, d)+ORDER BY x.d, x.a+statement okDROP TABLE limit_zero_outer;