Apache DataFusion SQL Query Engine
fix: request a null-aware mark join wherever a NULL mark is observable (#25560)
Adrian Garcia Badaraccob2a2c64a52ef
diff --git a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q08.benchmark b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q08.benchmarkindex 139c5b2a44..7100b8f84b 100644--- a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q08.benchmark+++ b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q08.benchmark@@ -23,11 +23,10 @@ WHERE o.z > 900SELECT i.id_n0 FROM small_inner i WHERE i.k = o.k AND i.z < o.z);-----# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.-false+trueexpect_plan HashJoinExec+expect_plan null_awarerun-- Q8: NOT IN correlated by both an equality and a non-equality, 50% NULL ondiff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rsindex 0ad8b44c40..135d1fc7f5 100644--- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs+++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs@@ -21,7 +21,6 @@ use std::ops::Deref;use std::sync::Arc;use crate::decorrelate::PullUpCorrelatedExpr;-use crate::extract_equijoin_predicate::split_eq_and_noneq_join_predicate;use crate::optimizer::ApplyOrder;use crate::utils::replace_qualified_name;use crate::{OptimizerConfig, OptimizerRule};@@ -81,7 +80,7 @@ impl OptimizerRule for DecorrelatePredicateSubquery {for expr in projection.expr {let original_name = expr.schema_name().to_string();let (new_input, mut rewritten_expr) =- rewrite_inner_subqueries(cur_input, expr, config, true)?;+ rewrite_inner_subqueries(cur_input, expr, config, true, true)?;if has_subquery(&rewritten_expr) {return Ok(Transformed::no(LogicalPlan::Projection(original_projection,@@ -136,7 +135,7 @@ impl OptimizerRule for DecorrelatePredicateSubquery {// The subquery expression is embedded within another expressionSubqueryPredicate::Embedded(expr) => {let (plan, expr_without_subqueries) =- rewrite_inner_subqueries(cur_input, expr, config, false)?;+ rewrite_filter_subqueries(cur_input, expr, config)?;cur_input = plan;other_exprs.push(expr_without_subqueries);}@@ -167,11 +166,61 @@ impl OptimizerRule for DecorrelatePredicateSubquery {}}+/// Rewrites the subqueries of one embedded `Filter` conjunct, deciding for each+/// occurrence whether its mark join has to be null-aware.+///+/// A `Filter` keeps a row only when the predicate is TRUE, and `AND`/`OR` make+/// TRUE only out of TRUE. A NULL mark therefore acts exactly like a FALSE mark+/// under them, so a non-negated `IN` reached through nothing but `AND`/`OR`+/// does not need the more expensive null-aware join. Every other context can+/// tell NULL from FALSE and gets one.+///+/// The decision is taken at the occurrence rather than once for the whole+/// conjunct, so a sibling's shape cannot change it, and any expression this+/// function does not name falls through to the null-aware branch.+fn rewrite_filter_subqueries(+ outer: LogicalPlan,+ expr: Expr,+ config: &dyn OptimizerConfig,+) -> Result<(LogicalPlan, Expr)> {+ match expr {+ Expr::BinaryExpr(BinaryExpr {+ left,+ op: op @ (Operator::And | Operator::Or),+ right,+ }) => {+ let (outer, left) = rewrite_filter_subqueries(outer, *left, config)?;+ let (outer, right) = rewrite_filter_subqueries(outer, *right, config)?;+ Ok((+ outer,+ Expr::BinaryExpr(BinaryExpr::new(Box::new(left), op, Box::new(right))),+ ))+ }+ Expr::InSubquery(in_subquery) => {+ // A subquery in the `IN` value is compared, not used as a filter+ // truth value, so its own mark is observable there.+ let needs_null_aware_mark =+ in_subquery.negated || has_subquery(&in_subquery.expr);+ rewrite_inner_subqueries(+ outer,+ Expr::InSubquery(in_subquery),+ config,+ false,+ needs_null_aware_mark,+ )+ }+ other => rewrite_inner_subqueries(outer, other, config, false, true),+ }+}++/// `needs_null_aware_mark` is `false` only when the caller can prove that a NULL+/// mark and a FALSE mark give the same answer. See [`rewrite_filter_subqueries`].fn rewrite_inner_subqueries(outer: LogicalPlan,expr: Expr,config: &dyn OptimizerConfig,materialize_in_value: bool,+ needs_null_aware_mark: bool,) -> Result<(LogicalPlan, Expr)> {let mut cur_input = outer;let alias = config.alias_generator();@@ -179,7 +228,14 @@ fn rewrite_inner_subqueries(Expr::Exists(Exists {subquery: Subquery { subquery, .. },negated,- }) => match mark_join(&cur_input, &subquery, None, negated, alias)? {+ }) => match mark_join(+ &cur_input,+ &subquery,+ None,+ negated,+ alias,+ needs_null_aware_mark,+ )? {Some((plan, exists_expr)) => {cur_input = plan;Ok(Transformed::yes(exists_expr))@@ -206,7 +262,14 @@ fn rewrite_inner_subqueries(.map_or(plan_err!("single expression required."), |output_expr| {Ok(Expr::eq(*expr.clone(), output_expr))})?;- mark_join(&cur_input, &subquery, Some(&in_predicate), negated, alias)?+ mark_join(+ &cur_input,+ &subquery,+ Some(&in_predicate),+ negated,+ alias,+ needs_null_aware_mark,+ )?};match rewritten {Some((plan, exists_expr)) => {@@ -234,7 +297,7 @@ fn in_subquery_value_mark_join(.map_or(plan_err!("single expression required."), Ok)?;let in_predicate = Expr::eq(expr.clone(), output_expr.clone());let Some((matched_plan, matched)) =- mark_join(left, subquery, Some(&in_predicate), false, alias)?+ mark_join(left, subquery, Some(&in_predicate), false, alias, true)?else {return Ok(None);};@@ -244,12 +307,12 @@ fn in_subquery_value_mark_join(.filter(output_expr.is_null())?.build()?;let Some((null_plan, subquery_has_null)) =- mark_join(&matched_plan, &null_subquery, None, false, alias)?+ mark_join(&matched_plan, &null_subquery, None, false, alias, true)?else {return Ok(None);};let Some((final_plan, subquery_non_empty)) =- mark_join(&null_plan, subquery, None, false, alias)?+ mark_join(&null_plan, subquery, None, false, alias, true)?else {return Ok(None);};@@ -371,6 +434,7 @@ fn build_join_top(in_predicate_opt.as_ref(),join_type,subquery_alias,+ true,)}@@ -395,16 +459,22 @@ fn mark_join(in_predicate_opt: Option<&Expr>,negated: bool,alias_generator: &Arc<AliasGenerator>,+ needs_null_aware_mark: bool,) -> Result<Option<(LogicalPlan, Expr)>> {let alias = alias_generator.next("__correlated_sq");let exists_col = Expr::Column(Column::new(Some(alias.clone()), "mark"));let exists_expr = if negated { !exists_col } else { exists_col };- Ok(- build_join(left, subquery, in_predicate_opt, JoinType::LeftMark, alias)?- .map(|plan| (plan, exists_expr)),- )+ Ok(build_join(+ left,+ subquery,+ in_predicate_opt,+ JoinType::LeftMark,+ alias,+ needs_null_aware_mark,+ )?+ .map(|plan| (plan, exists_expr)))}/// Check if join keys in the join filter may contain NULL values@@ -440,12 +510,25 @@ fn join_keys_may_be_null(Ok(false)}+/// Combines the `IN`/`NOT IN` equality with the correlation predicates, if any.+///+/// The equality stays the leading conjunct so that it becomes `on[0]`, the key+/// position a null-aware hash join reads as the `NOT IN` value key (see+/// `HashJoinExec::null_aware`).+fn in_predicate_first(in_predicate: Expr, correlation: Option<Expr>) -> Expr {+ match correlation {+ Some(correlation) => in_predicate.and(correlation),+ None => in_predicate,+ }+}+fn build_join(left: &LogicalPlan,subquery: &LogicalPlan,in_predicate_opt: Option<&Expr>,join_type: JoinType,alias: String,+ needs_null_aware_mark: bool,) -> Result<Option<LogicalPlan>> {let mut pull_up = PullUpCorrelatedExpr::new().with_in_predicate_opt(in_predicate_opt.cloned())@@ -471,29 +554,15 @@ fn build_join(replace_qualified_name(filter, &all_correlated_cols, &alias).map(Some)})?;- // The outer value expression of an `IN`/`NOT IN` predicate whose join filter- // is nothing but that predicate, recorded together with the subquery column- // it is compared against and a name for the column it can be projected as.- // Correlated subqueries are excluded on purpose: their correlation predicate- // is a second join key, and null-aware hash joins accept only a single key.+ // The outer value expression of an `IN`/`NOT IN` predicate, recorded together+ // with the subquery column it is compared against, a name for the column it+ // can be projected as, and the correlation predicates that share the join+ // filter with it.let mut in_value_expr = None;let mut join_filter = match (join_filter_opt, in_predicate_opt.cloned()) {(- Some(join_filter),- Some(Expr::BinaryExpr(BinaryExpr {- left,- op: Operator::Eq,- right,- })),- ) => {- let right_col = create_col_from_scalar_expr(&right, alias)?;- let in_predicate = Expr::eq(left.deref().clone(), Expr::Column(right_col));- in_predicate.and(join_filter)- }- (Some(join_filter), _) => join_filter,- (- _,+ correlation_opt,Some(Expr::BinaryExpr(BinaryExpr {left,op: Operator::Eq,@@ -503,14 +572,34 @@ fn build_join(let value_name = format!("{alias}_value");let right_col = create_col_from_scalar_expr(&right, alias)?;let value = left.deref().clone();- in_value_expr = Some((value.clone(), right_col.clone(), value_name));+ let in_predicate = Expr::eq(value.clone(), Expr::Column(right_col.clone()));+ let join_filter = in_predicate_first(in_predicate, correlation_opt.clone());+ in_value_expr = Some((value, right_col, value_name, correlation_opt));- Expr::eq(value, Expr::Column(right_col))+ join_filter}+ (Some(join_filter), _) => join_filter,(None, None) => lit(true),_ => return Ok(None),};+ // Whether this join needs null-aware (`NOT IN` three-valued) semantics.+ // Decided once: the constant projection below, the mark branch and the anti+ // branch all depend on the same answer. `NOT EXISTS` is two-valued and has+ // no `in_predicate_opt`, so it never qualifies; non-nullable keys on both+ // sides mean no NULL can reach the join in the first place.+ //+ // The left projection below only adds a non-nullable literal column and the+ // right projection only drops unreferenced columns, so neither changes the+ // nullability this looks at.+ let null_aware = in_predicate_opt.is_some()+ && match join_type {+ JoinType::LeftAnti => true,+ JoinType::LeftMark => needs_null_aware_mark,+ _ => false,+ }+ && join_keys_may_be_null(&join_filter, left.schema(), sub_query_alias.schema())?;+// `<constant> IN/NOT IN (<subquery>)`: the outer value expression holds no// column reference, so `<constant> = __correlated_sq.col` is not a valid// equi-join key (see `find_valid_equijoin_key_pair`) and stays in the join@@ -518,13 +607,16 @@ fn build_join(// right-only, so `push_down_filter` moves it into the subquery and drops the// very NULLs that make `NOT IN` UNKNOWN, and a join without equi-join keys// is planned as a nested loop join, which has no null-aware implementation.+ // A correlated subquery hits a third problem: its correlation predicate is a+ // valid equi-join key, so it takes `on[0]`, the position the null-aware hash+ // join reads as the `NOT IN` value key.// Projecting the constant as a column of the outer side turns the predicate- // into a real equi-join key so the null-aware hash join handles it.+ // into a real equi-join key, which fixes all three. It only pays for itself+ // on a join that ends up null-aware.let mut projected_left = None;- if let Some((value, right_col, mut value_name)) = in_value_expr+ if let Some((value, right_col, mut value_name, correlation_opt)) = in_value_expr&& value.column_refs().is_empty()- && matches!(join_type, JoinType::LeftAnti | JoinType::LeftMark)- && join_keys_may_be_null(&join_filter, left.schema(), sub_query_alias.schema())?+ && null_aware{// The projected column is unqualified, so a left field that already has// this name — however unlikely — would make the reference ambiguous.@@ -544,9 +636,9 @@ fn build_join(.project(projections)?.build()?,);- // `in_value_expr` is only set when the `IN` equality is the whole join- // filter, so it can simply be rebuilt against the projected column.- join_filter = Expr::eq(Expr::Column(value_col), Expr::Column(right_col));+ // Rebuild the `IN` equality against the projected column.+ let in_predicate = Expr::eq(Expr::Column(value_col), Expr::Column(right_col));+ join_filter = in_predicate_first(in_predicate, correlation_opt);}let left = projected_left.as_ref().unwrap_or(left);@@ -584,31 +676,10 @@ fn build_join(sub_query_alias.clone()};- let mark_filter_is_hashable_only =- if join_type == JoinType::LeftMark && in_predicate_opt.is_some() {- let (_, residual_filter) = split_eq_and_noneq_join_predicate(- join_filter.clone(),- left.schema(),- right_projected.schema(),- )?;- residual_filter.is_none()- } else {- false- };-// For scalar NOT IN mark joins, propagate null-aware semantics into the- // nullable mark column when the predicate can be implemented by hash keys.- // Non-equality correlated filters stay on the legacy path because hash join- // execution cannot mark UNKNOWN candidates for residual predicates.- let null_aware = join_type == JoinType::LeftMark- && in_predicate_opt.is_some()- && mark_filter_is_hashable_only- && join_keys_may_be_null(- &join_filter,- left.schema(),- right_projected.schema(),- )?;-+ // nullable mark column. A non-equality correlation stays behind as a+ // join filter, which the hash join also applies when it decides+ // whether a NULL makes the mark UNKNOWN.let new_plan = LogicalPlanBuilder::from(left.clone()).join_detailed_with_options(right_projected,@@ -628,18 +699,6 @@ fn build_join(return Ok(Some(new_plan));}- // Determine if this should be a null-aware anti join- // Null-aware semantics are only needed for NOT IN subqueries, not NOT EXISTS:- // - NOT IN: Uses three-valued logic, requires null-aware handling- // - NOT EXISTS: Uses two-valued logic, regular anti join is correct- // We can distinguish them: NOT IN has in_predicate_opt, NOT EXISTS does not- //- // Additionally, if the join keys are non-nullable on both sides, we don't need- // null-aware semantics because NULLs cannot exist in the data.- let null_aware = join_type == JoinType::LeftAnti- && in_predicate_opt.is_some()- && join_keys_may_be_null(&join_filter, left.schema(), sub_query_alias.schema())?;-// join our sub query into the main planlet new_plan = if null_aware {// Use join_detailed_with_options to set null_aware flag@@ -756,18 +815,6 @@ mod tests {plan.inputs().into_iter().any(has_null_aware_left_mark_join)}- fn has_non_null_aware_left_mark_join(plan: &LogicalPlan) -> bool {- if let LogicalPlan::Join(join) = plan- && join.join_type == JoinType::LeftMark- {- return !join.null_aware;- }-- plan.inputs()- .into_iter()- .any(has_non_null_aware_left_mark_join)- }-fn optimize_with_decorrelate(plan: LogicalPlan) -> Result<LogicalPlan> {let optimizer = crate::Optimizer::with_rules(vec![Arc::new(DecorrelatePredicateSubquery::new(),@@ -1495,39 +1542,6 @@ mod tests {)}- /// The same rewrite must not fire for a correlated subquery: the- /// correlation predicate is a second equi-join key, and null-aware hash- /// joins accept only one.- #[test]- fn constant_not_in_correlated_subquery_is_not_rewritten() -> Result<()> {- let outer_scan = nullable_scalar_mark_scan("outer_t")?;- let inner_scan = nullable_scalar_mark_scan("inner_t")?;-- let subquery = Arc::new(- LogicalPlanBuilder::from(inner_scan)- .filter(- out_ref_col(DataType::Int32, "outer_t.grp").eq(col("inner_t.grp")),- )?- .project(vec![col("inner_t.id")])?- .build()?,- );-- let plan = LogicalPlanBuilder::from(outer_scan)- .filter(not_in_subquery(lit(3i32), subquery))?- .build()?;-- assert_optimized_plan_equal!(- plan,- @r"- LeftAnti Join: Filter: Int32(3) = __correlated_sq_1.id AND outer_t.grp = __correlated_sq_1.grp null_aware [id:Int32;N, grp:Int32;N]- TableScan: outer_t [id:Int32;N, grp:Int32;N]- SubqueryAlias: __correlated_sq_1 [id:Int32;N, grp:Int32;N]- Projection: inner_t.id, inner_t.grp [id:Int32;N, grp:Int32;N]- TableScan: inner_t [id:Int32;N, grp:Int32;N]- "- )- }-#[test]fn correlated_not_in_mark_join_is_null_aware_for_hashable_filter() -> Result<()> {let outer_scan = nullable_scalar_mark_scan("outer_t")?;@@ -1556,34 +1570,6 @@ mod tests {Ok(())}- #[test]- fn correlated_not_in_mark_join_is_not_null_aware_for_residual_filter() -> Result<()> {- let outer_scan = nullable_scalar_mark_scan("outer_t")?;- let inner_scan = nullable_scalar_mark_scan("inner_t")?;-- let subquery = Arc::new(- LogicalPlanBuilder::from(inner_scan)- .filter(- out_ref_col(DataType::Int32, "outer_t.grp").lt(col("inner_t.grp")),- )?- .project(vec![col("inner_t.id")])?- .build()?,- );-- let plan = LogicalPlanBuilder::from(outer_scan)- .filter(not_in_subquery(col("outer_t.id"), subquery).is_null())?- .build()?;-- let optimized = optimize_with_decorrelate(plan)?;- assert!(- has_non_null_aware_left_mark_join(&optimized),- "{}",- optimized.display_indent_schema()- );-- Ok(())- }-#[test]fn in_subquery_both_side_expr() -> Result<()> {let table_scan = test_table_scan()?;diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.sltindex 050f223f9b..33e6099900 100644--- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt+++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt@@ -655,11 +655,15 @@ CREATE TABLE naconst_corr_t2(id INT, g INT) AS VALUES (1, 1), (NULL, 2);# A non-equality correlation stays in the join filter. Row (1, 1) sees the NULL# of (NULL, 2), so its `NOT IN` is UNKNOWN; row (2, 2) sees an empty subquery,# so its `NOT IN` is TRUE. Expected results verified with DuckDB.-query error DataFusion error: Error during planning: null_aware LeftAnti join requires equi\-join keys, but the join has none+query ISELECT id FROM naconst_corr_t1 WHERE 3 NOT IN (SELECT id FROM naconst_corr_t2 WHERE naconst_corr_t2.g > naconst_corr_t1.g) ORDER BY id;+----+2-query error DataFusion error: Error during planning: null_aware LeftAnti join requires equi\-join keys, but the join has none+query ISELECT id FROM naconst_corr_t1 WHERE NOT (3 IN (SELECT id FROM naconst_corr_t2 WHERE naconst_corr_t2.g > naconst_corr_t1.g)) ORDER BY id;+----+2statement okDROP TABLE naconst_corr_t1;@@ -683,17 +687,17 @@ query ISELECT z FROM naconst_corr_t3 WHERE 1 NOT IN (SELECT naconst_corr_t4.id FROM naconst_corr_t4 WHERE naconst_corr_t4.z = naconst_corr_t3.z) ORDER BY z;----7-10+NULL-# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.query ISELECT z FROM naconst_corr_t3 WHERE 2 NOT IN (SELECT naconst_corr_t4.id FROM naconst_corr_t4 WHERE naconst_corr_t4.z = naconst_corr_t3.z) ORDER BY z;-----10+NULL-query error DataFusion error: Error during planning: null_aware LeftAnti join requires equi\-join keys, but the join has none+query ISELECT z FROM naconst_corr_t3 WHERE 1 NOT IN (SELECT naconst_corr_t4.id FROM naconst_corr_t4 WHERE naconst_corr_t4.z > naconst_corr_t3.z) ORDER BY z;+----+NULL# The projected value column must be the FIRST equi-join key: a null-aware hash# join reads `on[0]` as the `NOT IN` value key and `on[1..]` as correlation@@ -702,16 +706,19 @@ query TTEXPLAIN SELECT z FROM naconst_corr_t3 WHERE 1 NOT IN (SELECT naconst_corr_t4.id FROM naconst_corr_t4 WHERE naconst_corr_t4.z = naconst_corr_t3.z);----logical_plan-01)LeftAnti Join: naconst_corr_t3.z = __correlated_sq_1.z Filter: Int64(1) = __correlated_sq_1.naconst_corr_t4.id null_aware-02)--TableScan: naconst_corr_t3 projection=[z]-03)--SubqueryAlias: __correlated_sq_1-04)----Projection: CAST(naconst_corr_t4.id AS Int64), naconst_corr_t4.z-05)------TableScan: naconst_corr_t4 projection=[id, z]+01)Projection: naconst_corr_t3.z+02)--LeftAnti Join: __correlated_sq_1_value = __correlated_sq_1.naconst_corr_t4.id, naconst_corr_t3.z = __correlated_sq_1.z null_aware+03)----Projection: naconst_corr_t3.z, Int64(1) AS __correlated_sq_1_value+04)------TableScan: naconst_corr_t3 projection=[z]+05)----SubqueryAlias: __correlated_sq_1+06)------Projection: CAST(naconst_corr_t4.id AS Int64), naconst_corr_t4.z+07)--------TableScan: naconst_corr_t4 projection=[id, z]physical_plan-01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(z@0, z@1)], filter=1 = naconst_corr_t4.id@0, null_aware-02)--DataSourceExec: partitions=1, partition_sizes=[1]-03)--ProjectionExec: expr=[CAST(id@0 AS Int64) as naconst_corr_t4.id, z@1 as z]-04)----DataSourceExec: partitions=1, partition_sizes=[1]+01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(__correlated_sq_1_value@1, naconst_corr_t4.id@0), (z@0, z@1)], projection=[z@0], null_aware+02)--ProjectionExec: expr=[z@0 as z, 1 as __correlated_sq_1_value]+03)----DataSourceExec: partitions=1, partition_sizes=[1]+04)--ProjectionExec: expr=[CAST(id@0 AS Int64) as naconst_corr_t4.id, z@1 as z]+05)----DataSourceExec: partitions=1, partition_sizes=[1]statement okDROP TABLE naconst_corr_t3;diff --git a/datafusion/sqllogictest/test_files/null_aware_mark_join.slt b/datafusion/sqllogictest/test_files/null_aware_mark_join.sltindex e4e14053c8..c45d87b51e 100644--- a/datafusion/sqllogictest/test_files/null_aware_mark_join.slt+++ b/datafusion/sqllogictest/test_files/null_aware_mark_join.slt@@ -663,23 +663,19 @@ CREATE TABLE nam_res_inner(id INT, z INT, g INT) AS VALUES(4, 35, 2),(NULL, 2, 2);-# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.query II rowsortSELECT id, z FROM nam_res_outerWHERE (id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.z < nam_res_outer.z)) IS NULL;----+2 20+NULL 30-# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.query II rowsortSELECT id, z FROM nam_res_outerWHERE (id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.z < nam_res_outer.z)) IS TRUE;-----2 205 1NULL 1-NULL 30query II rowsortSELECT id, z FROM nam_res_outer@@ -689,36 +685,33 @@ WHERE (id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.z < nam_res_outer.z))4 40# `NOT mark` must stay NULL for UNKNOWN rows instead of turning into TRUE.-# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.query II rowsortSELECT id, z FROM nam_res_outerWHERE NOT (id IN (SELECT i.id FROM nam_res_inner i WHERE i.z < nam_res_outer.z)) OR id = 4;-----2 204 405 1NULL 1-NULL 30# Positive IN goes through the same mark.-# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.query II rowsortSELECT id, z FROM nam_res_outerWHERE (id IN (SELECT i.id FROM nam_res_inner i WHERE i.z > nam_res_outer.z)) IS NULL;----+1 10+2 20+4 40+5 1+NULL 1+NULL 30# Equality and non-equality correlation together.-# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.query II rowsortSELECT id, z FROM nam_res_outerWHERE (id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.g = nam_res_outer.g AND i.z < nam_res_outer.z)) IS NULL;----+NULL 30-# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.query II rowsortSELECT id, z FROM nam_res_outerWHERE (id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.g = nam_res_outer.g AND i.z < nam_res_outer.z)) IS TRUE;@@ -726,7 +719,6 @@ WHERE (id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.g = nam_res_outer.g A2 205 1NULL 1-NULL 30# Equality correlation only (already null-aware; control).query II rowsort@@ -774,8 +766,6 @@ physical_plan# The negated form of the same query stays null-aware, because `NOT mark` tells# NULL from FALSE.-# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.query TTEXPLAIN SELECT id FROM nam_res_outer oWHERE o.z > 35 OR o.id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.z < o.z);@@ -783,7 +773,7 @@ WHERE o.z > 35 OR o.id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.z < o.z)logical_plan01)Projection: o.id02)--Filter: o.z > Int32(35) OR NOT __correlated_sq_1.mark-03)----LeftMark Join: o.id = __correlated_sq_1.id Filter: __correlated_sq_1.z < o.z+03)----LeftMark Join: o.id = __correlated_sq_1.id Filter: __correlated_sq_1.z < o.z null_aware04)------SubqueryAlias: o05)--------TableScan: nam_res_outer projection=[id, z]06)------SubqueryAlias: __correlated_sq_1@@ -792,7 +782,7 @@ logical_planphysical_plan01)FilterExec: z@1 > 35 OR NOT mark@2, projection=[id@0]02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-03)----HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(id@0, id@0)], filter=z@1 < z@0+03)----HashJoinExec: mode=CollectLeft, join_type=LeftMark, on=[(id@0, id@0)], filter=z@1 < z@0, null_aware04)------DataSourceExec: partitions=1, partition_sizes=[1]05)------DataSourceExec: partitions=1, partition_sizes=[1]@@ -801,12 +791,12 @@ physical_planstatement okSET datafusion.execution.batch_size = 1;-# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.query II rowsortSELECT id, z FROM nam_res_outerWHERE (id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.z < nam_res_outer.z)) IS NULL;----+2 20+NULL 30statement okRESET datafusion.execution.batch_size;@@ -925,8 +915,6 @@ CREATE TABLE nam_guard_inner(id INT, z INT) AS VALUES (1,5), (NULL,50);# A non-negated `IN` reached through `OR` keeps the same rows with a NULL mark# and with a FALSE mark, so the join must NOT be null-aware. It is then free to# swap sides to `RightMark` as well.-# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.query TTEXPLAIN SELECT id FROM nam_guard_outerWHERE id IN (SELECT i.id FROM nam_guard_inner i) OR z > 25;@@ -934,7 +922,7 @@ WHERE id IN (SELECT i.id FROM nam_guard_inner i) OR z > 25;logical_plan01)Projection: nam_guard_outer.id02)--Filter: __correlated_sq_1.mark OR nam_guard_outer.z > Int32(25)-03)----LeftMark Join: nam_guard_outer.id = __correlated_sq_1.id null_aware+03)----LeftMark Join: nam_guard_outer.id = __correlated_sq_1.id04)------TableScan: nam_guard_outer projection=[id, z]05)------SubqueryAlias: __correlated_sq_106)--------SubqueryAlias: i@@ -942,7 +930,7 @@ logical_planphysical_plan01)FilterExec: mark@2 OR z@1 > 25, projection=[id@0]02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-03)----HashJoinExec: mode=CollectLeft, join_type=LeftMark, on=[(id@0, id@0)], null_aware+03)----HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(id@0, id@0)]04)------DataSourceExec: partitions=1, partition_sizes=[1]05)------DataSourceExec: partitions=1, partition_sizes=[1]@@ -950,8 +938,6 @@ physical_plan# two-valued, so its mark is never NULL and it cannot make the `IN` mark# observable. Treating it as negative cost this query the plain join and the# `RightMark` swap.-# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.query TTEXPLAIN SELECT id FROM nam_guard_outerWHERE id IN (SELECT i.id FROM nam_guard_inner i)@@ -962,7 +948,7 @@ logical_plan02)--Filter: __correlated_sq_1.mark OR NOT __correlated_sq_2.mark03)----Projection: nam_guard_outer.id, __correlated_sq_1.mark, __correlated_sq_2.mark04)------LeftMark Join: Filter: __correlated_sq_2.z > nam_guard_outer.z-05)--------LeftMark Join: nam_guard_outer.id = __correlated_sq_1.id null_aware+05)--------LeftMark Join: nam_guard_outer.id = __correlated_sq_1.id06)----------TableScan: nam_guard_outer projection=[id, z]07)----------SubqueryAlias: __correlated_sq_108)------------SubqueryAlias: i@@ -975,15 +961,13 @@ physical_plan02)--NestedLoopJoinExec: join_type=RightMark, filter=z@1 > z@0, projection=[id@0, mark@2, mark@3]03)----DataSourceExec: partitions=1, partition_sizes=[1]04)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-05)------HashJoinExec: mode=CollectLeft, join_type=LeftMark, on=[(id@0, id@0)], null_aware+05)------HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(id@0, id@0)]06)--------DataSourceExec: partitions=1, partition_sizes=[1]07)--------DataSourceExec: partitions=1, partition_sizes=[1]# A constant value expression is projected as an outer column only when the# join ends up null-aware. This mark join does not, so the constant stays a# filter and no `__correlated_sq_*_value` column is produced.-# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.query TTEXPLAIN SELECT id FROM nam_guard_outerWHERE 3 IN (SELECT i.id FROM nam_guard_inner i) OR z > 25;@@ -991,22 +975,23 @@ WHERE 3 IN (SELECT i.id FROM nam_guard_inner i) OR z > 25;logical_plan01)Projection: nam_guard_outer.id02)--Filter: __correlated_sq_1.mark OR nam_guard_outer.z > Int32(25)-03)----Projection: nam_guard_outer.id, nam_guard_outer.z, __correlated_sq_1.mark-04)------LeftMark Join: __correlated_sq_1_value = __correlated_sq_1.i.id null_aware-05)--------Projection: nam_guard_outer.id, nam_guard_outer.z, Int64(3) AS __correlated_sq_1_value-06)----------TableScan: nam_guard_outer projection=[id, z]-07)--------SubqueryAlias: __correlated_sq_1-08)----------Projection: CAST(i.id AS Int64)-09)------------SubqueryAlias: i-10)--------------TableScan: nam_guard_inner projection=[id]+03)----LeftMark Join:+04)------TableScan: nam_guard_outer projection=[id, z]+05)------SubqueryAlias: __correlated_sq_1+06)--------Projection: CAST(i.id AS Int64)+07)----------SubqueryAlias: i+08)------------Filter: nam_guard_inner.id = Int32(3)+09)--------------TableScan: nam_guard_inner projection=[id]physical_plan01)FilterExec: mark@2 OR z@1 > 25, projection=[id@0]02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-03)----HashJoinExec: mode=CollectLeft, join_type=LeftMark, on=[(__correlated_sq_1_value@2, i.id@0)], projection=[id@0, z@1, mark@3], null_aware-04)------ProjectionExec: expr=[id@0 as id, z@1 as z, 3 as __correlated_sq_1_value]-05)--------DataSourceExec: partitions=1, partition_sizes=[1]-06)------ProjectionExec: expr=[CAST(id@0 AS Int64) as i.id]-07)--------DataSourceExec: partitions=1, partition_sizes=[1]+03)----NestedLoopJoinExec: join_type=RightMark+04)------CoalescePartitionsExec+05)--------ProjectionExec: expr=[CAST(id@0 AS Int64) as i.id]+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+07)------------FilterExec: id@0 = 3+08)--------------DataSourceExec: partitions=1, partition_sizes=[1]+09)------DataSourceExec: partitions=1, partition_sizes=[1]statement okDROP TABLE nam_guard_outer;diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex ca0b0b1ff8..c208d7ab19 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -730,8 +730,6 @@ logical_plan06)--EmptyRelation: rows=1#simple_uncorrelated_scalar_subquery2-# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.query TTexplain select (select count(*) from t1) as b, (select count(1) from t2)----@@ -1254,8 +1252,6 @@ where t1.t1_id > 40 or t1.t1_id in (select t2.t2_id from t2 where t1.t1_int > 0)44 d 4# not_in_subquery_to_join_with_correlated_outer_filter_disjunction-# Pinned to today's behaviour, which is incorrect. See-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.query TTexplain select t1.t1_id,t1.t1_name,@@ -1266,7 +1262,7 @@ where t1.t1_id = 11 or t1.t1_id + 12 not in (select t2.t2_id + 1 from t2 where tlogical_plan01)Projection: t1.t1_id, t1.t1_name, t1.t1_int02)--Filter: t1.t1_id = Int32(11) OR NOT __correlated_sq_1.mark-03)----LeftMark Join: CAST(t1.t1_id AS Int64) + Int64(12) = __correlated_sq_1.t2.t2_id + Int64(1) Filter: t1.t1_int > Int32(0)+03)----LeftMark Join: CAST(t1.t1_id AS Int64) + Int64(12) = __correlated_sq_1.t2.t2_id + Int64(1) Filter: t1.t1_int > Int32(0) null_aware04)------TableScan: t1 projection=[t1_id, t1_name, t1_int]05)------SubqueryAlias: __correlated_sq_106)--------Projection: CAST(t2.t2_id AS Int64) + Int64(1)