Apache DataFusion SQL Query Engine
fix: correlated exists/not exists subqueries hit the count bug for groupless aggregates (#25391)
Mohammad Naqvi86e10a5499b1
diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rsindex 8b9d25e2fb..44e4fb4415 100644--- a/datafusion/expr/src/utils.rs+++ b/datafusion/expr/src/utils.rs@@ -1596,6 +1596,38 @@ pub fn collect_subquery_cols(})}+/// Determine the subquery-side expressions a join filter compares,+/// stopping at the first subquery-referencing sub-expression rather than+/// descending to the bare columns inside of it.+///+/// `t1.a = t2.b` yields `t2.b`, matching [`collect_subquery_cols`]. But+/// `t1.a = CAST(t2.b AS INT)` yields `CAST(t2.b AS INT)` whole, not `t2.b`.+/// The two differ whenever the correlated predicate wraps the subquery's+/// column in an expression, in which case we need to evaluate the full+/// expression before grouping, so that two column values which compare+/// equal after the wrapping expression end up in the same group.+pub fn collect_subquery_join_exprs(+ exprs: &[Expr],+ subquery_schema: &DFSchema,+) -> Result<Vec<Expr>> {+ let mut result: Vec<Expr> = vec![];+ for expr in exprs {+ expr.apply(|e| {+ if e.column_refs()+ .into_iter()+ .all(|col| subquery_schema.has_column(col))+ && !e.column_refs().is_empty()+ && !result.contains(e)+ {+ result.push(e.clone());+ return Ok(TreeNodeRecursion::Jump);+ }+ Ok(TreeNodeRecursion::Continue)+ })?;+ }+ Ok(result)+}+#[cfg(test)]mod tests {use super::*;diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rsindex 0a1db9b212..7dcf8fde93 100644--- a/datafusion/optimizer/src/decorrelate.rs+++ b/datafusion/optimizer/src/decorrelate.rs@@ -426,6 +426,50 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {None};+ if self.exists_sub_query+ && self.pull_up_having_expr.is_some()+ && let Some(expr) = conjunction(subquery_filters.clone())+ {+ let unqualified_expr = expr+ .transform_up(|e| {+ if let Expr::Column(Column { name, .. }) = &e {+ Ok(Transformed::yes(Expr::Column(+ Column::new_unqualified(name),+ )))+ } else {+ Ok(Transformed::no(e))+ }+ })+ .data()?;+ let combined = match self.pull_up_having_expr.take() {+ Some(existing) => existing.and(unqualified_expr),+ None => unqualified_expr,+ };+ self.pull_up_having_expr = Some(combined);+ let new_plan =+ LogicalPlanBuilder::from((*plan_filter.input).clone()).build()?;+ let mut carried_correlated_cols = correlated_subquery_cols;+ if let Some(existing) =+ self.correlated_subquery_cols_map.get(&*plan_filter.input)+ {+ carried_correlated_cols.extend(existing.iter().cloned());+ }+ self.correlated_subquery_cols_map+ .insert(new_plan.clone(), carried_correlated_cols);+ if !expr_result_map_for_count_bug.is_empty() {+ self.collected_count_expr_map+ .insert(new_plan.clone(), expr_result_map_for_count_bug);+ } else if let Some(input_map) = self+ .collected_count_expr_map+ .get(&*plan_filter.input)+ .cloned()+ {+ self.collected_count_expr_map+ .insert(new_plan.clone(), input_map);+ }+ return Ok(Transformed::yes(new_plan));+ }+match (&pull_up_expr_opt, &self.pull_up_having_expr) {(Some(_), Some(_)) => {// Error path@@ -464,15 +508,18 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {&mut local_correlated_cols,);// add missing columns to Projection- let mut missing_exprs =- self.collect_missing_exprs(&projection.expr, &local_correlated_cols)?;+ let mut missing_exprs = self.collect_missing_exprs(+ &projection.expr,+ &local_correlated_cols,+ projection.input.schema(),+ )?;let mut expr_result_map_for_count_bug = HashMap::new();if let Some(expr_result_map) =self.collected_count_expr_map.get(&*projection.input){proj_exprs_evaluation_result_on_empty_batch(- &projection.expr,+ &missing_exprs,projection.input.schema(),expr_result_map,&mut expr_result_map_for_count_bug,@@ -511,7 +558,6 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {&self.correlated_subquery_cols_map,&mut local_correlated_cols,);-// A grouping set that omits a column the pull up adds either// keeps its sets as they are (every set already groups by the// column), or gains the column in every set, `ROLLUP(i.k)`@@ -544,6 +590,7 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {self.collect_missing_exprs(&aggregate.group_expr,&local_correlated_cols,+ aggregate.input.schema(),)?}GroupingSetsPullUp::Unsafe => {@@ -558,6 +605,7 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {self.collect_missing_exprs(&aggregate.group_expr,&local_correlated_cols,+ aggregate.input.schema(),)?}}@@ -566,6 +614,7 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {self.collect_missing_exprs(&aggregate.group_expr,&local_correlated_cols,+ aggregate.input.schema(),)?};@@ -800,9 +849,10 @@ impl PullUpCorrelatedExpr {}fn collect_missing_exprs(- &self,+ &mut self,exprs: &[Expr],correlated_subquery_cols: &BTreeSet<Column>,+ _subquery_schema: &DFSchemaRef,) -> Result<Vec<Expr>> {let mut missing_exprs =Vec::with_capacity(exprs.len() + correlated_subquery_cols.len());@@ -813,19 +863,22 @@ impl PullUpCorrelatedExpr {}for col in correlated_subquery_cols.iter() {let col_expr = Expr::Column(col.clone());- if !missing_exprs.contains(&col_expr) {+ if !collides_with_existing(&missing_exprs, &col_expr) {missing_exprs.push(col_expr)}}if let Some(pull_up_having) = &self.pull_up_having_expr {- let filter_apply_columns = pull_up_having.column_refs();- for col in filter_apply_columns {- // add to missing_exprs if not already there- let contains = missing_exprs+ for col in pull_up_having.column_refs() {+ let col_expr = Expr::Column(col.clone());+ // `col` is unqualified but the projection may already have+ // it as `agg.c`. Check by bare name so that doesn't get+ // added twice.+ let already_present = missing_exprs.iter()- .any(|expr| matches!(expr, Expr::Column(c) if c == col));- if !contains {- missing_exprs.push(Expr::Column(col.clone()))+ .any(|expr| matches!(expr, Expr::Column(c) if c.name == col.name))+ || collides_with_existing(&missing_exprs, &col_expr);+ if !already_present {+ missing_exprs.push(col_expr)}}}@@ -925,6 +978,16 @@ fn partitions_by_all(expr: &Expr, cols: &BTreeSet<Column>) -> bool {})}+/// True if `candidate` collides with an existing entry's schema name,+/// e.g. a bare column already wrapped in an equally-named `CAST`.+/// Colliding means `LogicalPlanBuilder::project` would reject it as a duplicate.+fn collides_with_existing(exprs: &[Expr], candidate: &Expr) -> bool {+ let candidate_name = candidate.schema_name().to_string();+ exprs+ .iter()+ .any(|expr| expr.schema_name().to_string() == candidate_name)+}+fn can_pullup_over_aggregation(expr: &Expr) -> bool {if let Expr::BinaryExpr(BinaryExpr {left,@@ -979,7 +1042,7 @@ fn remove_duplicated_filter(Expr::BinaryExpr(b) => b.op.swap() == Some(b.op),_ => true,},- "remove_duplicated_filter: in_predicate must use a commutative operator"+ "in_predicate must use a commutative operator");Ok(filters@@ -1105,9 +1168,14 @@ fn filter_exprs_evaluation_result_on_empty_batch(let simplifier = ExprSimplifier::new(info);let result_expr = simplifier.simplify(result_expr)?;match &result_expr {- // evaluate to false or null on empty batch, no need to pull upExpr::Literal(ScalarValue::Null, _)- | Expr::Literal(ScalarValue::Boolean(Some(false)), _) => None,+ | Expr::Literal(ScalarValue::Boolean(None), _)+ | Expr::Literal(ScalarValue::Boolean(Some(false)), _) => {+ for (name, exprs) in input_expr_result_map_for_count_bug {+ expr_result_map_for_count_bug.insert(name.clone(), exprs.clone());+ }+ None+ }// evaluate to true on empty batch, need to pull up the exprExpr::Literal(ScalarValue::Boolean(Some(true)), _) => {for (name, exprs) in input_expr_result_map_for_count_bug {diff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rsindex 99b57a5494..9c20108da2 100644--- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs+++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs@@ -20,14 +20,17 @@ use std::collections::BTreeSet;use std::ops::Deref;use std::sync::Arc;-use crate::decorrelate::PullUpCorrelatedExpr;+use crate::analyzer::type_coercion::TypeCoercionRewriter;+use crate::decorrelate::{PullUpCorrelatedExpr, UN_MATCHED_ROW_INDICATOR};use crate::extract_equijoin_predicate::split_eq_and_noneq_join_predicate;use crate::optimizer::ApplyOrder;use crate::utils::replace_qualified_name;use crate::{OptimizerConfig, OptimizerRule};use datafusion_common::alias::AliasGenerator;-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};+use datafusion_common::tree_node::{+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,+};use datafusion_common::{Column, DFSchema, NullEquality, Result, ScalarValue, assert_or_internal_err,internal_datafusion_err, internal_err, not_impl_err, plan_err,@@ -760,6 +763,30 @@ fn with_null_aware_value_keys())}+fn subquery_operands_are_bare_columns(+ join_filters: &[Expr],+ subquery_cols: &BTreeSet<Column>,+) -> bool {+ join_filters.iter().all(|filter| {+ let mut bare = true;+ filter+ .apply(|e| {+ let refs = e.column_refs();+ if !refs.is_empty() && refs.iter().all(|c| subquery_cols.contains(*c)) {+ bare = matches!(e, Expr::Column(_));+ return Ok(if bare {+ TreeNodeRecursion::Jump+ } else {+ TreeNodeRecursion::Stop+ });+ }+ Ok(TreeNodeRecursion::Continue)+ })+ .expect("infallible");+ bare+ })+}+/// The outcome of [`build_join`].struct BuiltJoin {/// The outer plan with the subquery joined into it.@@ -777,11 +804,48 @@ fn build_join(alias: &str,needs_null_aware_mark: bool,) -> Result<Option<BuiltJoin>> {- let mut pull_up = PullUpCorrelatedExpr::new()- .with_in_predicate_opt(in_predicate_opt.cloned())- .with_exists_sub_query(in_predicate_opt.is_none());-- let new_plan = subquery.clone().rewrite(&mut pull_up).data()?;+ let exists_sub_query = in_predicate_opt.is_none();+ let rewrite =+ |handle_count_bug: bool| -> Result<(LogicalPlan, PullUpCorrelatedExpr)> {+ let mut pull_up = PullUpCorrelatedExpr::new()+ .with_in_predicate_opt(in_predicate_opt.cloned())+ .with_exists_sub_query(exists_sub_query)+ .with_need_handle_count_bug(handle_count_bug);+ let new_plan = subquery.clone().rewrite(&mut pull_up).data()?;+ Ok((new_plan, pull_up))+ };+ let (mut new_plan, mut pull_up) = rewrite(exists_sub_query)?;++ let mut count_bug_compensation = pull_up+ .collected_count_expr_map+ .get(&new_plan)+ .filter(|m| !m.is_empty())+ .cloned();+ // A HAVING-failure miss looks identical to an absent group once compensated, so bail if the join filter reads the aggregate's own output.+ let reads_agg_output = count_bug_compensation.as_ref().is_some_and(|expr_map| {+ pull_up+ .join_filters+ .iter()+ .flat_map(|f| f.column_refs())+ .any(|c| expr_map.contains_key(&c.name))+ });+ let subquery_cols: BTreeSet<Column> = pull_up+ .correlated_subquery_cols_map+ .values()+ .flatten()+ .cloned()+ .collect();+ let keys_are_bare = count_bug_compensation.is_none()+ || subquery_operands_are_bare_columns(&pull_up.join_filters, &subquery_cols);+ // Fall back to the uncompensated rewrite whenever compensation can't safely apply, so a HAVING never gets silently dropped.+ let compensable = pull_up.can_pull_up+ && !reads_agg_output+ && keys_are_bare+ && (count_bug_compensation.is_some() || pull_up.pull_up_having_expr.is_none());+ if exists_sub_query && !compensable {+ (new_plan, pull_up) = rewrite(false)?;+ count_bug_compensation = None;+ }if !pull_up.can_pull_up {return Ok(None);}@@ -801,6 +865,26 @@ fn build_join(replace_qualified_name(filter, &all_correlated_cols, alias).map(Some)})?;+ if let Some(expr_map) = count_bug_compensation+ && !expr_map.is_empty()+ {+ return build_join_with_count_bug(+ left,+ sub_query_alias,+ join_filter_opt,+ join_type,+ alias,+ &expr_map,+ pull_up.pull_up_having_expr.as_ref(),+ )+ .map(|plan| {+ Some(BuiltJoin {+ plan,+ mark_is_three_valued_exact: false,+ })+ });+ }+// The two sides of the `IN` predicate: the value from the outer plan and// the subquery output it is compared with, renamed to the alias. A// multi-column `(a, b) IN (SELECT x, y ...)` has one pair per tuple@@ -1059,6 +1143,116 @@ fn build_join(}))}+/// Builds the join for a correlated `EXISTS` subquery whose groupless+/// aggregate requires count-bug compensation.+fn build_join_with_count_bug(+ left: &LogicalPlan,+ sub_query_alias: LogicalPlan,+ join_filter_opt: Option<Expr>,+ join_type: JoinType,+ alias: &str,+ expr_map: &crate::decorrelate::ExprResultMap,+ pull_up_having_expr: Option<&Expr>,+) -> Result<LogicalPlan> {+ let pull_up_having_expr = pull_up_having_expr+ .map(|f| {+ f.clone()+ .transform_up(|e| match e {+ Expr::Column(c) => Ok(Transformed::yes(Expr::Column(Column::new(+ Some(alias),+ c.name,+ )))),+ _ => Ok(Transformed::no(e)),+ })+ .data()+ })+ .transpose()?;+ let pull_up_having_expr = pull_up_having_expr.as_ref();++ let joined = LogicalPlanBuilder::from(left.clone())+ .join_on(sub_query_alias, JoinType::Left, join_filter_opt)?+ .build()?;++ let left_projection: Vec<Expr> = left+ .schema()+ .columns()+ .into_iter()+ .map(Expr::from)+ .collect();++ let indicator_col = Expr::Column(Column::new(Some(alias), UN_MATCHED_ROW_INDICATOR));++ let having_arm = pull_up_having_expr.map(|f| f.clone().is_not_true());++ let mut expr_rewrite = TypeCoercionRewriter::new(joined.schema());++ // The subquery's HAVING clause, evaluated against an unmatched row's+ // default aggregate values (e.g. count(*) defaults to 0, sum(x) to+ // NULL), is usually `true`, but not once a second filter is combined+ // into it, so it cannot be assumed.+ let unmatched_having_default = match pull_up_having_expr {+ Some(f) => f+ .clone()+ .transform_up(|e| {+ if let Expr::Column(Column { name, .. }) = &e+ && let Some(default_value) = expr_map.get(name)+ {+ return Ok(Transformed::yes(default_value.clone()));+ }+ Ok(Transformed::no(e))+ })+ .data()+ .and_then(|simplified| simplified.rewrite(&mut expr_rewrite).data())?,+ None => lit(true),+ };++ // EXISTS is true by default (the groupless aggregate always+ // produces a row), unless one of three cases holds:+ // (1) the outer row joined against an inner group whose HAVING+ // predicate failed,+ // (2) an unmatched outer row's own default aggregate values fail+ // that same HAVING clause.+ let exists_expr = match having_arm {+ Some(when_expr) => when(+ indicator_col.clone().is_null(),+ unmatched_having_default.is_true(),+ )+ .when(when_expr, lit(false))+ .otherwise(lit(true))?+ .rewrite(&mut expr_rewrite)+ .data()?,+ None => lit(true),+ };++ let new_plan = match join_type {+ JoinType::LeftMark | JoinType::RightMark => {+ let mut proj_exprs = left_projection;+ proj_exprs.push(exists_expr.alias_qualified(Some(alias), "mark"));+ LogicalPlanBuilder::from(joined)+ .project(proj_exprs)?+ .build()?+ }+ JoinType::LeftAnti => LogicalPlanBuilder::from(joined)+ .filter(not(exists_expr))?+ .project(left_projection)?+ .build()?,+ JoinType::LeftSemi => LogicalPlanBuilder::from(joined)+ .filter(exists_expr)?+ .project(left_projection)?+ .build()?,+ _ => {+ return internal_err!("unsupported join type {join_type:?}");+ }+ };++ debug!(+ "predicate subquery (count bug) optimized:\n{}",+ new_plan.display_indent()+ );++ Ok(new_plan)+}+#[derive(Debug)]struct SubqueryInfo {query: Subquery,@@ -1111,6 +1305,7 @@ mod tests {TableScanBuilder, and, binary_expr, col, cube, grouping_set, out_ref_col, rollup,table_scan,};+ use datafusion_functions_aggregate::count::count_udaf;macro_rules! assert_optimized_plan_equal {(@@ -1438,7 +1633,7 @@ mod tests {SubqueryAlias: __correlated_sq_2 [o_custkey:Int64]Projection: orders.o_custkey [o_custkey:Int64]TableScan: orders [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]- "+ ")}@@ -1666,7 +1861,7 @@ mod tests {assert_optimized_plan_equal!(plan,- @r"+ @"Projection: customer.c_custkey [c_custkey:Int64]LeftSemi Join: Filter: customer.c_custkey = __correlated_sq_1.o_custkey AND (customer.c_custkey = __correlated_sq_1.o_custkey OR __correlated_sq_1.o_orderkey = Int32(1)) [c_custkey:Int64, c_name:Utf8]TableScan: customer [c_custkey:Int64, c_name:Utf8]@@ -2722,7 +2917,7 @@ mod tests {assert_optimized_plan_equal!(plan,- @r"+ @"Projection: customer.c_custkey [c_custkey:Int64]LeftSemi Join: Filter: customer.c_custkey = __correlated_sq_1.o_custkey OR __correlated_sq_1.o_orderkey = Int32(1) [c_custkey:Int64, c_name:Utf8]TableScan: customer [c_custkey:Int64, c_name:Utf8]@@ -2855,6 +3050,37 @@ mod tests {)}+ /// Test for correlated exists subquery filter with disjunction and count bug+ #[test]+ fn exists_subquery_disjunction_with_count_bug() -> Result<()> {+ let sq = Arc::new(+ LogicalPlanBuilder::from(test_table_scan_with_name("sq")?)+ .filter(out_ref_col(DataType::UInt32, "test.a").eq(col("sq.a")))?+ .aggregate(Vec::<Expr>::new(), vec![count_udaf().call(vec![])])?+ .build()?,+ );++ let plan = LogicalPlanBuilder::from(test_table_scan()?)+ .filter(exists(sq).or(col("test.c").eq(lit(1))))?+ .project(vec![col("test.b")])?+ .build()?;++ assert_optimized_plan_equal!(+ plan,+ @r"+ Projection: test.b [b:UInt32]+ Projection: test.a, test.b, test.c [a:UInt32, b:UInt32, c:UInt32]+ Filter: __correlated_sq_1.mark OR test.c = Int32(1) [a:UInt32, b:UInt32, c:UInt32, mark:Boolean]+ Projection: test.a, test.b, test.c, Boolean(true) AS mark [a:UInt32, b:UInt32, c:UInt32, mark:Boolean]+ Left Join: Filter: test.a = __correlated_sq_1.a [a:UInt32, b:UInt32, c:UInt32, a:UInt32;N, __always_true:Boolean;N, count():Int64;N]+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]+ SubqueryAlias: __correlated_sq_1 [a:UInt32, __always_true:Boolean, count():Int64]+ Aggregate: groupBy=[[sq.a, Boolean(true) AS __always_true]], aggr=[[count()]] [a:UInt32, __always_true:Boolean, count():Int64]+ TableScan: sq [a:UInt32, b:UInt32, c:UInt32]+ "+ )+ }+/// Test for correlated EXISTS subquery filter#[test]fn exists_subquery_correlated() -> Result<()> {@@ -2987,7 +3213,7 @@ mod tests {assert_optimized_plan_equal!(plan,- @r"+ @"Projection: test.b [b:UInt32]LeftSemi Join: Filter: UInt32(1) + __correlated_sq_1.a > test.a * UInt32(2) [a:UInt32, b:UInt32, c:UInt32]TableScan: test [a:UInt32, b:UInt32, c:UInt32]@@ -3046,7 +3272,7 @@ mod tests {assert_optimized_plan_equal!(plan,- @r"+ @"Projection: test.b [b:UInt32]LeftSemi Join: Filter: UInt32(1) + __correlated_sq_1.a > test.a * UInt32(2) [a:UInt32, b:UInt32, c:UInt32]TableScan: test [a:UInt32, b:UInt32, c:UInt32]@@ -3077,7 +3303,7 @@ mod tests {assert_optimized_plan_equal!(plan,- @r"+ @"Projection: test.b [b:UInt32]LeftSemi Join: Filter: UInt32(1) + __correlated_sq_1.a > test.a * UInt32(2) [a:UInt32, b:UInt32, c:UInt32]TableScan: test [a:UInt32, b:UInt32, c:UInt32]@@ -3108,7 +3334,7 @@ mod tests {assert_optimized_plan_equal!(plan,- @r"+ @"Projection: test.b [b:UInt32]LeftSemi Join: Filter: UInt32(1) + __correlated_sq_1.a > test.a * UInt32(2) [a:UInt32, b:UInt32, c:UInt32]TableScan: test [a:UInt32, b:UInt32, c:UInt32]diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex 39b3961c82..3cd3bfbf19 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -1294,6 +1294,19 @@ select t1.t1_int from t1 where (24+# correlated_scalar_subquery_having_default_not_coerced_to_match_stacked_filter+query II rowsort+SELECT t1_id, (+ SELECT c FROM (+ SELECT count(*) AS c FROM t2 WHERE t2.t2_id = t1.t1_id HAVING count(*) < 10+ ) s WHERE c > 5+) AS v FROM t1+----+11 NULL+22 NULL+33 NULL+44 NULL+#correlated_scalar_subquery_count_agg_in_case_whenquery TTexplain@@ -1857,6 +1870,275 @@ item1 1970-01-01T00:00:03 75statement okdrop table source_table;+++# correlated_exists_groupless_count_agg_count_bug+query II rowsort+SELECT t1_id, t1_int FROM t1 WHERE EXISTS (SELECT count(*) FROM t2 WHERE t1.t1_id = t2.t2_id)+----+11 1+22 2+33 3+44 4++# correlated_exists_groupless_count_agg_count_bug_having+query TT+explain SELECT t1_id, t1_int FROM t1 WHERE EXISTS (SELECT count(*) FROM t2 WHERE t1.t1_id = t2.t2_id HAVING count(*) = 0)+----+logical_plan+01)Projection: t1.t1_id, t1.t1_int+02)--Filter: CASE WHEN __correlated_sq_1.__always_true IS NULL THEN Boolean(true) WHEN __correlated_sq_1.count(Int64(1)) = Int64(0) IS NOT TRUE THEN Boolean(false) ELSE Boolean(true) END+03)----Projection: t1.t1_id, t1.t1_int, __correlated_sq_1.count(Int64(1)), __correlated_sq_1.__always_true+04)------Left Join: t1.t1_id = __correlated_sq_1.t2_id+05)--------TableScan: t1 projection=[t1_id, t1_int]+06)--------SubqueryAlias: __correlated_sq_1+07)----------Projection: t2.t2_id, count(Int64(1)), Boolean(true) AS __always_true+08)------------Aggregate: groupBy=[[t2.t2_id]], aggr=[[count(Int64(1))]]+09)--------------TableScan: t2 projection=[t2_id]++query II+SELECT t1_id, t1_int FROM t1 WHERE EXISTS (SELECT count(*) FROM t2 WHERE t1.t1_id = t2.t2_id HAVING count(*) = 0)+----+33 3++# correlated_exists_groupless_compound_having_select_dummy_column+query II+SELECT t1_id, t1_int FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE t1.t1_id = t2.t2_id HAVING count(*) = 0 AND count(*) < 100)+----+33 3++# correlated_not_exists_groupless_compound_having_select_dummy_column+query II rowsort+SELECT t1_id, t1_int FROM t1 WHERE NOT EXISTS (SELECT 1 FROM t2 WHERE t1.t1_id = t2.t2_id HAVING count(*) = 0 AND count(*) < 100)+----+11 1+22 2+44 4++# correlated_exists_groupless_having_reads_outer_column_falls_back+query I+SELECT t1_id FROM t1 WHERE EXISTS (+ SELECT count(*) FROM t2 WHERE t2.t2_id = t1.t1_id HAVING count(*) = t1.t1_int+)+----+11++# correlated_exists_groupless_having_with_stacked_filter_via_cte+query II+SELECT t1_id, t1_int FROM t1 WHERE EXISTS (+ WITH agg AS (SELECT count(*) AS c FROM t2 WHERE t1.t1_id = t2.t2_id HAVING count(*) = 0)+ SELECT c FROM agg WHERE c >= 0+)+----+33 3++# correlated_not_exists_groupless_having_with_stacked_filter_via_cte+query II rowsort+SELECT t1_id, t1_int FROM t1 WHERE NOT EXISTS (+ WITH agg AS (SELECT count(*) AS c FROM t2 WHERE t1.t1_id = t2.t2_id HAVING count(*) = 0)+ SELECT c FROM agg WHERE c >= 0+)+----+11 1+22 2+44 4++# correlated_exists_groupless_having_with_stacked_filter_on_different_aggregate+query II+SELECT t1_id, t1_int FROM t1 WHERE EXISTS (+ WITH agg AS (SELECT count(*) AS c, sum(t2_int) AS s FROM t2 WHERE t1.t1_id = t2.t2_id HAVING count(*) = 0)+ SELECT c FROM agg WHERE s > 5+)+----++# correlated_not_exists_groupless_count_agg_count_bug+query II+SELECT t1_id, t1_int FROM t1 WHERE NOT EXISTS (SELECT count(*) FROM t2 WHERE t1.t1_id = t2.t2_id)+----++# correlated_exists_groupless_null_default_agg_always_true+query II rowsort+SELECT t1_id, t1_int FROM t1 WHERE EXISTS (SELECT sum(t2_int) FROM t2 WHERE t1.t1_id = t2.t2_id)+----+11 1+22 2+33 3+44 4++# correlated_not_exists_groupless_null_default_agg_always_true+query II+SELECT t1_id, t1_int FROM t1 WHERE NOT EXISTS (SELECT sum(t2_int) FROM t2 WHERE t1.t1_id = t2.t2_id)+----++# correlated_not_exists_groupless_null_default_having_is_null_not_false+query I rowsort+SELECT t1_id FROM t1 WHERE NOT EXISTS (+ SELECT c FROM (+ SELECT count(*) AS c, sum(t2_int) AS s FROM t2+ WHERE t1.t1_id = t2.t2_id HAVING count(*) = 0+ ) agg WHERE s > 5+)+----+11+22+33+44++# correlated_not_exists_groupless_null_default_having_is_null_not_false_embedded_in_or+query I rowsort+SELECT t1_id FROM t1 WHERE t1_id > 100 OR NOT EXISTS (+ SELECT c FROM (+ SELECT count(*) AS c, sum(t2_int) AS s FROM t2+ WHERE t1.t1_id = t2.t2_id HAVING count(*) = 0+ ) agg WHERE s > 5+)+----+11+22+33+44++# correlated_exists_groupless_count_agg_count_bug_embedded_in_or+query I rowsort+select t1_id from t1+where t1_id > 40 or exists (select count(*) from t2 where t1.t1_id = t2.t2_id having count(*) = 0)+----+33+44++# correlated_not_exists_groupless_count_agg_count_bug_embedded_in_or+query I rowsort+select t1_id from t1+where t1_id > 40 or not exists (select count(*) from t2 where t1.t1_id = t2.t2_id having count(*) = 0)+----+11+22+44++# correlated_exists_groupless_null_default_agg_count_bug_embedded_in_or+query I rowsort+select t1_id from t1+where t1_id > 40 or exists (select sum(t2_int) from t2 where t1.t1_id = t2.t2_id having sum(t2_int) is null)+----+33+44++# correlated_not_exists_groupless_null_default_agg_count_bug_embedded_in_or+query I rowsort+select t1_id from t1+where t1_id > 40 or not exists (select sum(t2_int) from t2 where t1.t1_id = t2.t2_id having sum(t2_int) is null)+----+11+22+44++statement ok+CREATE TABLE dedup_outer(k INT) AS VALUES (1), (2);++statement ok+CREATE TABLE dedup_inner(k TEXT, v INT) AS VALUES ('1', 10), ('1', 20), ('01', 30), ('01', 40);++# TODO(#26014): should match DuckDB (1, 2), not main's answer.+# correlated_exists_groupless_count_agg_duplicate_match_dedup+query I rowsort+SELECT o.k FROM dedup_outer o WHERE EXISTS (+ SELECT count(*) FROM dedup_inner i WHERE CAST(i.k AS INT) = o.k+)+----+1++# TODO(#26014): should match DuckDB, not main's answer.+# correlated_not_exists_groupless_count_agg_duplicate_match_dedup+query I rowsort+SELECT o.k FROM dedup_outer o WHERE NOT EXISTS (+ SELECT count(*) FROM dedup_inner i WHERE CAST(i.k AS INT) = o.k HAVING count(*) = 0+)+----+1+2++statement ok+DROP TABLE dedup_outer;++statement ok+DROP TABLE dedup_inner;++statement ok+CREATE TABLE cte_dedup_outer(k INT) AS VALUES (1), (2), (3);++statement ok+CREATE TABLE cte_dedup_inner(k TEXT, v INT) AS VALUES ('1', 10), ('01', 20), ('2', 30);++# TODO(#26014): should match DuckDB, not main's answer.+# correlated_exists_groupless_count_agg_duplicate_match_dedup_via_cte+query I+SELECT o.k FROM cte_dedup_outer o WHERE EXISTS (+ WITH agg AS (SELECT count(*) AS c FROM cte_dedup_inner i WHERE CAST(i.k AS INT) = o.k)+ SELECT c FROM agg WHERE c = 2+)+----++# TODO(#26014): should match DuckDB, not main's answer.+# correlated_not_exists_groupless_count_agg_duplicate_match_dedup_via_cte+query I rowsort+SELECT o.k FROM cte_dedup_outer o WHERE NOT EXISTS (+ WITH agg AS (SELECT count(*) AS c FROM cte_dedup_inner i WHERE CAST(i.k AS INT) = o.k)+ SELECT c FROM agg WHERE c = 2+)+----+1+2+3++statement ok+DROP TABLE cte_dedup_outer;++statement ok+DROP TABLE cte_dedup_inner;++statement ok+CREATE TABLE limit_zero_outer(k INT) AS VALUES (1);++statement ok+CREATE TABLE limit_zero_inner(k INT, v INT) AS VALUES (1, NULL);++# correlated_exists_groupless_count_agg_limit_zero+query I+SELECT o.k FROM limit_zero_outer o WHERE EXISTS (+ SELECT count(*) FROM limit_zero_inner i WHERE i.k = o.k LIMIT 0+)+----++# correlated_not_exists_groupless_count_agg_limit_zero+query I+SELECT o.k FROM limit_zero_outer o WHERE NOT EXISTS (+ SELECT count(*) FROM limit_zero_inner i WHERE i.k = o.k LIMIT 0+)+----+1++# correlated_in_groupless_max_agg_is_not_count_bug_compensated+query I+SELECT t1_id FROM t1+WHERE t1_int + 2 IN (SELECT max(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id)+----+11++# correlated_not_in_groupless_count_agg_is_not_count_bug_compensated+query I rowsort+SELECT t1_id FROM t1+WHERE t1_int NOT IN (SELECT count(*) FROM t2 WHERE t2.t2_id = t1.t1_id)+----+22+33+44++statement ok+DROP TABLE limit_zero_outer;++statement ok+DROP TABLE limit_zero_inner;+statement count 0drop table t1;