Skip to content

Apache DataFusion SQL Query Engine

fix: return a query error for correlated scalar subquery with outer-column HAVING over count aggregates (#26723)

siddu9d15cf079fe9
diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs
index 5143ac87c1..80500b89bf 100644
--- a/datafusion/optimizer/src/decorrelate.rs
+++ b/datafusion/optimizer/src/decorrelate.rs
@@ -68,6 +68,14 @@ pub struct PullUpCorrelatedExpr {
pub collected_count_expr_map: HashMap<LogicalPlan, ExprResultMap>,
/// pull up having expr, which must be evaluated after the Join
pub pull_up_having_expr: Option<Expr>,
+ /// Outer-reference conditions from a `Filter` that sits directly above a
+ /// count-bug `Aggregate`: a `HAVING` clause (or an outer-only filter)
+ /// that filters the aggregate *output*. Unlike `join_filters`, these do
+ /// not merely correlate the two sides: an unmatched LEFT JOIN row may
+ /// mean one of them rejected the group rather than that no group
+ /// matched, and a scalar subquery must yield NULL for a rejected group.
+ /// `build_join` uses them to reject such queries up front.
+ pub having_like_join_filters: Vec<Expr>,
/// whether we have converted a scalar aggregation into a group aggregation. When unnesting
/// lateral joins, we need to produce a left outer join in such cases.
///
@@ -124,6 +132,7 @@ impl PullUpCorrelatedExpr {
need_handle_count_bug: false,
collected_count_expr_map: HashMap::new(),
pull_up_having_expr: None,
+ having_like_join_filters: Vec::new(),
pulled_up_scalar_agg: false,
correlated_filters: Vec::new(),
column_refs_above: Vec::new(),
@@ -415,6 +424,22 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
// in_predicate may be already included in the join filters, remove it from the join filters first.
join_filters = remove_duplicated_filter(join_filters, in_predicate)?;
}
+ // A `Filter` directly above a count-bug `Aggregate` is a
+ // HAVING clause (or an outer-only filter above the aggregate):
+ // its outer-reference conditions filter the aggregate *output*,
+ // so a LEFT JOIN row can be unmatched either because the
+ // correlation matched no group or because one of them rejected
+ // the group. Record them so the scalar rewrite can tell the
+ // two apart; where the output needs count-bug compensation it
+ // rejects the query instead of guessing. A `Filter` below the
+ // aggregate only prunes input rows and is left out.
+ if self
+ .collected_count_expr_map
+ .contains_key(&*plan_filter.input)
+ {
+ self.having_like_join_filters
+ .extend(join_filters.iter().cloned());
+ }
let correlated_subquery_cols =
collect_subquery_cols(&join_filters, subquery_schema)?;
for expr in join_filters {
diff --git a/datafusion/optimizer/src/scalar_subquery_to_join.rs b/datafusion/optimizer/src/scalar_subquery_to_join.rs
index 126b97aec3..7dac77768c 100644
--- a/datafusion/optimizer/src/scalar_subquery_to_join.rs
+++ b/datafusion/optimizer/src/scalar_subquery_to_join.rs
@@ -349,6 +349,35 @@ fn build_join(
.alias(subquery_alias.to_string())?
.build()?;
+ // A HAVING or outer-only filter recorded from above the aggregate cannot
+ // be rewritten into the join when the output column gets a compensation
+ // CASE below. Kept in the ON, a row the condition rejects looks unmatched
+ // and falls back to the empty-input default, which is the issue's wrong 0. Checked
+ // in the compensation CASE instead, the condition would have to be
+ // evaluated twice, against the defaults for an unmatched row and against
+ // the actual output for a matched one, and neither evaluation is right
+ // for the rows the other one covers. The gate looks at the output column
+ // alone because the map also holds aggregates that appear only in the
+ // HAVING, whose NULL defaults need no CASE; an output whose own default
+ // is NULL (the subquery projects `max`, say) keeps filtering in the ON as
+ // before, which is correct there because an unmatched row is NULL either
+ // way. Rather than guess, reject the query.
+ let output_name = subquery_plan.schema().field(0).name();
+ let output_needs_compensation = match collected_count_expr_map
+ .as_ref()
+ .and_then(|expr_map| expr_map.get(output_name))
+ {
+ Some(default_expr) => {
+ !evaluates_to_null(default_expr.clone(), default_expr.column_refs())?
+ }
+ None => false,
+ };
+ if output_needs_compensation && !pull_up.having_like_join_filters.is_empty() {
+ return plan_err!(
+ "Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns, and this rewrite cannot evaluate it against both the group and the empty input without risking wrong results. Move the condition outside the subquery"
+ );
+ }
+
let join_filter_opt = build_join_filter(
pull_up.join_filters,
pull_up.correlated_subquery_cols_map.values().flatten(),
diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt
index 4b553cc9aa..8a6143bc68 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -2219,6 +2219,71 @@ WHERE t1_int + 2 NOT IN (SELECT max(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id)
query error Physical plan does not support logical expression InSubquery
SELECT t1_id, t1_int + 2 IN (SELECT max(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id) FROM t1
+# https://github.com/apache/datafusion/issues/26503
+# A correlated scalar subquery whose HAVING reads an outer column is rejected:
+# left in the join ON it returns the empty-input 0 where NULL is correct.
+statement error .*Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns
+SELECT x.a, x.d,
+ (SELECT count(*) FROM (VALUES (1)) AS y(c) WHERE y.c = x.d HAVING count(*) >= x.a) AS v
+FROM (VALUES (1, 1), (2, 1), (1, 9)) AS x(a, d)
+ORDER BY x.d, x.a
+
+# Same shape with the predicate in a filter above the aggregate rather than in
+# HAVING: the filter references only an outer column, so it is rejected too.
+statement error .*Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns
+SELECT (SELECT n FROM (SELECT count(*) AS n FROM (VALUES (1)) AS y(c) WHERE y.c = x.d) WHERE x.a > 1) AS v
+FROM (VALUES (1, 1)) AS x(a, d)
+
+# A local HAVING that holds on the empty-input default still yields the
+# default (0), not NULL, when no inner row matches the correlation. Only
+# conditions that read outer columns are rejected.
+query I
+SELECT (SELECT count(*) FROM (VALUES (1)) AS y(c) WHERE y.c = x.d HAVING count(*) >= 0) AS v
+FROM (VALUES (1, 9)) AS x(a, d)
+----
+0
+
+# A local HAVING that fails on the empty-input default yields NULL.
+query I
+SELECT (SELECT count(*) FROM (VALUES (1)) AS y(c) WHERE y.c = x.d HAVING count(*) > 0) AS v
+FROM (VALUES (1, 9)) AS x(a, d)
+----
+NULL
+
+# A stacked filter above the aggregate that reads an outer column is rejected
+# for the same reason, whether or not a HAVING keyword is written.
+statement error .*Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns
+SELECT x.n, x.d,
+ (SELECT s.n FROM (SELECT count(*) AS n FROM (VALUES (1)) AS y(c) WHERE y.c = x.d) AS s
+ WHERE s.n < x.n) AS v
+FROM (VALUES (5, 9), (0, 9), (5, 1)) AS x(n, d)
+ORDER BY x.d, x.n
+
+# A group the HAVING rejects would fall back to the empty-input default when
+# the condition stays in the ON, so this shape is rejected as well.
+statement error .*Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns
+SELECT x.a, x.d,
+ (SELECT count(*) FROM (VALUES (1)) AS y(c) WHERE y.c = x.d HAVING count(*) < x.a) AS v
+FROM (VALUES (1, 1), (2, 1), (1, 9)) AS x(a, d)
+ORDER BY x.d, x.a
+
+# max appears only in the HAVING, but the subquery outputs count(*), so the
+# output still needs compensation and the query is rejected.
+statement error .*Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns
+SELECT x.a, x.d,
+ (SELECT count(*) FROM (VALUES (1)) AS y(c) WHERE y.c = x.d HAVING max(y.c) < x.a) AS v
+FROM (VALUES (1, 1), (2, 1), (1, 9)) AS x(a, d)
+ORDER BY x.d, x.a
+
+# The same shape, but here the HAVING does hold on the empty input
+# (max(y.c) IS NULL). The join cannot tell which side decides the unmatched
+# row, so this one is rejected too.
+statement error .*Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns
+SELECT x.a, x.d,
+ (SELECT count(*) FROM (VALUES (1)) AS y(c) WHERE y.c = x.d HAVING max(y.c) IS NULL OR count(*) < x.a) AS v
+FROM (VALUES (1, 1), (2, 1), (1, 9)) AS x(a, d)
+ORDER BY x.d, x.a
+
statement ok
DROP TABLE limit_zero_outer;