Skip to content

Apache DataFusion SQL Query Engine

fix: request a null-aware mark join wherever a NULL mark is observable (#25560)

Adrian Garcia Badaraccob2a2c64a52ef
diff --git a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q08.benchmark b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q08.benchmark
index 139c5b2a44..7100b8f84b 100644
--- a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q08.benchmark
+++ b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q08.benchmark
@@ -23,11 +23,10 @@ WHERE o.z > 900
SELECT i.id_n0 FROM small_inner i WHERE i.k = o.k AND i.z < o.z
);
----
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
-false
+true
expect_plan HashJoinExec
+expect_plan null_aware
run
-- Q8: NOT IN correlated by both an equality and a non-equality, 50% NULL on
diff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
index 0ad8b44c40..135d1fc7f5 100644
--- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
+++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
@@ -21,7 +21,6 @@ use std::ops::Deref;
use std::sync::Arc;
use crate::decorrelate::PullUpCorrelatedExpr;
-use crate::extract_equijoin_predicate::split_eq_and_noneq_join_predicate;
use crate::optimizer::ApplyOrder;
use crate::utils::replace_qualified_name;
use crate::{OptimizerConfig, OptimizerRule};
@@ -81,7 +80,7 @@ impl OptimizerRule for DecorrelatePredicateSubquery {
for expr in projection.expr {
let original_name = expr.schema_name().to_string();
let (new_input, mut rewritten_expr) =
- rewrite_inner_subqueries(cur_input, expr, config, true)?;
+ rewrite_inner_subqueries(cur_input, expr, config, true, true)?;
if has_subquery(&rewritten_expr) {
return Ok(Transformed::no(LogicalPlan::Projection(
original_projection,
@@ -136,7 +135,7 @@ impl OptimizerRule for DecorrelatePredicateSubquery {
// The subquery expression is embedded within another expression
SubqueryPredicate::Embedded(expr) => {
let (plan, expr_without_subqueries) =
- rewrite_inner_subqueries(cur_input, expr, config, false)?;
+ rewrite_filter_subqueries(cur_input, expr, config)?;
cur_input = plan;
other_exprs.push(expr_without_subqueries);
}
@@ -167,11 +166,61 @@ impl OptimizerRule for DecorrelatePredicateSubquery {
}
}
+/// Rewrites the subqueries of one embedded `Filter` conjunct, deciding for each
+/// occurrence whether its mark join has to be null-aware.
+///
+/// A `Filter` keeps a row only when the predicate is TRUE, and `AND`/`OR` make
+/// TRUE only out of TRUE. A NULL mark therefore acts exactly like a FALSE mark
+/// under them, so a non-negated `IN` reached through nothing but `AND`/`OR`
+/// does not need the more expensive null-aware join. Every other context can
+/// tell NULL from FALSE and gets one.
+///
+/// The decision is taken at the occurrence rather than once for the whole
+/// conjunct, so a sibling's shape cannot change it, and any expression this
+/// function does not name falls through to the null-aware branch.
+fn rewrite_filter_subqueries(
+ outer: LogicalPlan,
+ expr: Expr,
+ config: &dyn OptimizerConfig,
+) -> Result<(LogicalPlan, Expr)> {
+ match expr {
+ Expr::BinaryExpr(BinaryExpr {
+ left,
+ op: op @ (Operator::And | Operator::Or),
+ right,
+ }) => {
+ let (outer, left) = rewrite_filter_subqueries(outer, *left, config)?;
+ let (outer, right) = rewrite_filter_subqueries(outer, *right, config)?;
+ Ok((
+ outer,
+ Expr::BinaryExpr(BinaryExpr::new(Box::new(left), op, Box::new(right))),
+ ))
+ }
+ Expr::InSubquery(in_subquery) => {
+ // A subquery in the `IN` value is compared, not used as a filter
+ // truth value, so its own mark is observable there.
+ let needs_null_aware_mark =
+ in_subquery.negated || has_subquery(&in_subquery.expr);
+ rewrite_inner_subqueries(
+ outer,
+ Expr::InSubquery(in_subquery),
+ config,
+ false,
+ needs_null_aware_mark,
+ )
+ }
+ other => rewrite_inner_subqueries(outer, other, config, false, true),
+ }
+}
+
+/// `needs_null_aware_mark` is `false` only when the caller can prove that a NULL
+/// mark and a FALSE mark give the same answer. See [`rewrite_filter_subqueries`].
fn rewrite_inner_subqueries(
outer: LogicalPlan,
expr: Expr,
config: &dyn OptimizerConfig,
materialize_in_value: bool,
+ needs_null_aware_mark: bool,
) -> Result<(LogicalPlan, Expr)> {
let mut cur_input = outer;
let alias = config.alias_generator();
@@ -179,7 +228,14 @@ fn rewrite_inner_subqueries(
Expr::Exists(Exists {
subquery: Subquery { subquery, .. },
negated,
- }) => match mark_join(&cur_input, &subquery, None, negated, alias)? {
+ }) => match mark_join(
+ &cur_input,
+ &subquery,
+ None,
+ negated,
+ alias,
+ needs_null_aware_mark,
+ )? {
Some((plan, exists_expr)) => {
cur_input = plan;
Ok(Transformed::yes(exists_expr))
@@ -206,7 +262,14 @@ fn rewrite_inner_subqueries(
.map_or(plan_err!("single expression required."), |output_expr| {
Ok(Expr::eq(*expr.clone(), output_expr))
})?;
- mark_join(&cur_input, &subquery, Some(&in_predicate), negated, alias)?
+ mark_join(
+ &cur_input,
+ &subquery,
+ Some(&in_predicate),
+ negated,
+ alias,
+ needs_null_aware_mark,
+ )?
};
match rewritten {
Some((plan, exists_expr)) => {
@@ -234,7 +297,7 @@ fn in_subquery_value_mark_join(
.map_or(plan_err!("single expression required."), Ok)?;
let in_predicate = Expr::eq(expr.clone(), output_expr.clone());
let Some((matched_plan, matched)) =
- mark_join(left, subquery, Some(&in_predicate), false, alias)?
+ mark_join(left, subquery, Some(&in_predicate), false, alias, true)?
else {
return Ok(None);
};
@@ -244,12 +307,12 @@ fn in_subquery_value_mark_join(
.filter(output_expr.is_null())?
.build()?;
let Some((null_plan, subquery_has_null)) =
- mark_join(&matched_plan, &null_subquery, None, false, alias)?
+ mark_join(&matched_plan, &null_subquery, None, false, alias, true)?
else {
return Ok(None);
};
let Some((final_plan, subquery_non_empty)) =
- mark_join(&null_plan, subquery, None, false, alias)?
+ mark_join(&null_plan, subquery, None, false, alias, true)?
else {
return Ok(None);
};
@@ -371,6 +434,7 @@ fn build_join_top(
in_predicate_opt.as_ref(),
join_type,
subquery_alias,
+ true,
)
}
@@ -395,16 +459,22 @@ fn mark_join(
in_predicate_opt: Option<&Expr>,
negated: bool,
alias_generator: &Arc<AliasGenerator>,
+ needs_null_aware_mark: bool,
) -> Result<Option<(LogicalPlan, Expr)>> {
let alias = alias_generator.next("__correlated_sq");
let exists_col = Expr::Column(Column::new(Some(alias.clone()), "mark"));
let exists_expr = if negated { !exists_col } else { exists_col };
- Ok(
- build_join(left, subquery, in_predicate_opt, JoinType::LeftMark, alias)?
- .map(|plan| (plan, exists_expr)),
- )
+ Ok(build_join(
+ left,
+ subquery,
+ in_predicate_opt,
+ JoinType::LeftMark,
+ alias,
+ needs_null_aware_mark,
+ )?
+ .map(|plan| (plan, exists_expr)))
}
/// Check if join keys in the join filter may contain NULL values
@@ -440,12 +510,25 @@ fn join_keys_may_be_null(
Ok(false)
}
+/// Combines the `IN`/`NOT IN` equality with the correlation predicates, if any.
+///
+/// The equality stays the leading conjunct so that it becomes `on[0]`, the key
+/// position a null-aware hash join reads as the `NOT IN` value key (see
+/// `HashJoinExec::null_aware`).
+fn in_predicate_first(in_predicate: Expr, correlation: Option<Expr>) -> Expr {
+ match correlation {
+ Some(correlation) => in_predicate.and(correlation),
+ None => in_predicate,
+ }
+}
+
fn build_join(
left: &LogicalPlan,
subquery: &LogicalPlan,
in_predicate_opt: Option<&Expr>,
join_type: JoinType,
alias: String,
+ needs_null_aware_mark: bool,
) -> Result<Option<LogicalPlan>> {
let mut pull_up = PullUpCorrelatedExpr::new()
.with_in_predicate_opt(in_predicate_opt.cloned())
@@ -471,29 +554,15 @@ fn build_join(
replace_qualified_name(filter, &all_correlated_cols, &alias).map(Some)
})?;
- // The outer value expression of an `IN`/`NOT IN` predicate whose join filter
- // is nothing but that predicate, recorded together with the subquery column
- // it is compared against and a name for the column it can be projected as.
- // Correlated subqueries are excluded on purpose: their correlation predicate
- // is a second join key, and null-aware hash joins accept only a single key.
+ // The outer value expression of an `IN`/`NOT IN` predicate, recorded together
+ // with the subquery column it is compared against, a name for the column it
+ // can be projected as, and the correlation predicates that share the join
+ // filter with it.
let mut in_value_expr = None;
let mut join_filter = match (join_filter_opt, in_predicate_opt.cloned()) {
(
- Some(join_filter),
- Some(Expr::BinaryExpr(BinaryExpr {
- left,
- op: Operator::Eq,
- right,
- })),
- ) => {
- let right_col = create_col_from_scalar_expr(&right, alias)?;
- let in_predicate = Expr::eq(left.deref().clone(), Expr::Column(right_col));
- in_predicate.and(join_filter)
- }
- (Some(join_filter), _) => join_filter,
- (
- _,
+ correlation_opt,
Some(Expr::BinaryExpr(BinaryExpr {
left,
op: Operator::Eq,
@@ -503,14 +572,34 @@ fn build_join(
let value_name = format!("{alias}_value");
let right_col = create_col_from_scalar_expr(&right, alias)?;
let value = left.deref().clone();
- in_value_expr = Some((value.clone(), right_col.clone(), value_name));
+ let in_predicate = Expr::eq(value.clone(), Expr::Column(right_col.clone()));
+ let join_filter = in_predicate_first(in_predicate, correlation_opt.clone());
+ in_value_expr = Some((value, right_col, value_name, correlation_opt));
- Expr::eq(value, Expr::Column(right_col))
+ join_filter
}
+ (Some(join_filter), _) => join_filter,
(None, None) => lit(true),
_ => return Ok(None),
};
+ // Whether this join needs null-aware (`NOT IN` three-valued) semantics.
+ // Decided once: the constant projection below, the mark branch and the anti
+ // branch all depend on the same answer. `NOT EXISTS` is two-valued and has
+ // no `in_predicate_opt`, so it never qualifies; non-nullable keys on both
+ // sides mean no NULL can reach the join in the first place.
+ //
+ // The left projection below only adds a non-nullable literal column and the
+ // right projection only drops unreferenced columns, so neither changes the
+ // nullability this looks at.
+ let null_aware = in_predicate_opt.is_some()
+ && match join_type {
+ JoinType::LeftAnti => true,
+ JoinType::LeftMark => needs_null_aware_mark,
+ _ => false,
+ }
+ && join_keys_may_be_null(&join_filter, left.schema(), sub_query_alias.schema())?;
+
// `<constant> IN/NOT IN (<subquery>)`: the outer value expression holds no
// column reference, so `<constant> = __correlated_sq.col` is not a valid
// equi-join key (see `find_valid_equijoin_key_pair`) and stays in the join
@@ -518,13 +607,16 @@ fn build_join(
// right-only, so `push_down_filter` moves it into the subquery and drops the
// very NULLs that make `NOT IN` UNKNOWN, and a join without equi-join keys
// is planned as a nested loop join, which has no null-aware implementation.
+ // A correlated subquery hits a third problem: its correlation predicate is a
+ // valid equi-join key, so it takes `on[0]`, the position the null-aware hash
+ // join reads as the `NOT IN` value key.
// Projecting the constant as a column of the outer side turns the predicate
- // into a real equi-join key so the null-aware hash join handles it.
+ // into a real equi-join key, which fixes all three. It only pays for itself
+ // on a join that ends up null-aware.
let mut projected_left = None;
- if let Some((value, right_col, mut value_name)) = in_value_expr
+ if let Some((value, right_col, mut value_name, correlation_opt)) = in_value_expr
&& value.column_refs().is_empty()
- && matches!(join_type, JoinType::LeftAnti | JoinType::LeftMark)
- && join_keys_may_be_null(&join_filter, left.schema(), sub_query_alias.schema())?
+ && null_aware
{
// The projected column is unqualified, so a left field that already has
// this name — however unlikely — would make the reference ambiguous.
@@ -544,9 +636,9 @@ fn build_join(
.project(projections)?
.build()?,
);
- // `in_value_expr` is only set when the `IN` equality is the whole join
- // filter, so it can simply be rebuilt against the projected column.
- join_filter = Expr::eq(Expr::Column(value_col), Expr::Column(right_col));
+ // Rebuild the `IN` equality against the projected column.
+ let in_predicate = Expr::eq(Expr::Column(value_col), Expr::Column(right_col));
+ join_filter = in_predicate_first(in_predicate, correlation_opt);
}
let left = projected_left.as_ref().unwrap_or(left);
@@ -584,31 +676,10 @@ fn build_join(
sub_query_alias.clone()
};
- let mark_filter_is_hashable_only =
- if join_type == JoinType::LeftMark && in_predicate_opt.is_some() {
- let (_, residual_filter) = split_eq_and_noneq_join_predicate(
- join_filter.clone(),
- left.schema(),
- right_projected.schema(),
- )?;
- residual_filter.is_none()
- } else {
- false
- };
-
// For scalar NOT IN mark joins, propagate null-aware semantics into the
- // nullable mark column when the predicate can be implemented by hash keys.
- // Non-equality correlated filters stay on the legacy path because hash join
- // execution cannot mark UNKNOWN candidates for residual predicates.
- let null_aware = join_type == JoinType::LeftMark
- && in_predicate_opt.is_some()
- && mark_filter_is_hashable_only
- && join_keys_may_be_null(
- &join_filter,
- left.schema(),
- right_projected.schema(),
- )?;
-
+ // nullable mark column. A non-equality correlation stays behind as a
+ // join filter, which the hash join also applies when it decides
+ // whether a NULL makes the mark UNKNOWN.
let new_plan = LogicalPlanBuilder::from(left.clone())
.join_detailed_with_options(
right_projected,
@@ -628,18 +699,6 @@ fn build_join(
return Ok(Some(new_plan));
}
- // Determine if this should be a null-aware anti join
- // Null-aware semantics are only needed for NOT IN subqueries, not NOT EXISTS:
- // - NOT IN: Uses three-valued logic, requires null-aware handling
- // - NOT EXISTS: Uses two-valued logic, regular anti join is correct
- // We can distinguish them: NOT IN has in_predicate_opt, NOT EXISTS does not
- //
- // Additionally, if the join keys are non-nullable on both sides, we don't need
- // null-aware semantics because NULLs cannot exist in the data.
- let null_aware = join_type == JoinType::LeftAnti
- && in_predicate_opt.is_some()
- && join_keys_may_be_null(&join_filter, left.schema(), sub_query_alias.schema())?;
-
// join our sub query into the main plan
let new_plan = if null_aware {
// Use join_detailed_with_options to set null_aware flag
@@ -756,18 +815,6 @@ mod tests {
plan.inputs().into_iter().any(has_null_aware_left_mark_join)
}
- fn has_non_null_aware_left_mark_join(plan: &LogicalPlan) -> bool {
- if let LogicalPlan::Join(join) = plan
- && join.join_type == JoinType::LeftMark
- {
- return !join.null_aware;
- }
-
- plan.inputs()
- .into_iter()
- .any(has_non_null_aware_left_mark_join)
- }
-
fn optimize_with_decorrelate(plan: LogicalPlan) -> Result<LogicalPlan> {
let optimizer = crate::Optimizer::with_rules(vec![Arc::new(
DecorrelatePredicateSubquery::new(),
@@ -1495,39 +1542,6 @@ mod tests {
)
}
- /// The same rewrite must not fire for a correlated subquery: the
- /// correlation predicate is a second equi-join key, and null-aware hash
- /// joins accept only one.
- #[test]
- fn constant_not_in_correlated_subquery_is_not_rewritten() -> Result<()> {
- let outer_scan = nullable_scalar_mark_scan("outer_t")?;
- let inner_scan = nullable_scalar_mark_scan("inner_t")?;
-
- let subquery = Arc::new(
- LogicalPlanBuilder::from(inner_scan)
- .filter(
- out_ref_col(DataType::Int32, "outer_t.grp").eq(col("inner_t.grp")),
- )?
- .project(vec![col("inner_t.id")])?
- .build()?,
- );
-
- let plan = LogicalPlanBuilder::from(outer_scan)
- .filter(not_in_subquery(lit(3i32), subquery))?
- .build()?;
-
- assert_optimized_plan_equal!(
- plan,
- @r"
- LeftAnti Join: Filter: Int32(3) = __correlated_sq_1.id AND outer_t.grp = __correlated_sq_1.grp null_aware [id:Int32;N, grp:Int32;N]
- TableScan: outer_t [id:Int32;N, grp:Int32;N]
- SubqueryAlias: __correlated_sq_1 [id:Int32;N, grp:Int32;N]
- Projection: inner_t.id, inner_t.grp [id:Int32;N, grp:Int32;N]
- TableScan: inner_t [id:Int32;N, grp:Int32;N]
- "
- )
- }
-
#[test]
fn correlated_not_in_mark_join_is_null_aware_for_hashable_filter() -> Result<()> {
let outer_scan = nullable_scalar_mark_scan("outer_t")?;
@@ -1556,34 +1570,6 @@ mod tests {
Ok(())
}
- #[test]
- fn correlated_not_in_mark_join_is_not_null_aware_for_residual_filter() -> Result<()> {
- let outer_scan = nullable_scalar_mark_scan("outer_t")?;
- let inner_scan = nullable_scalar_mark_scan("inner_t")?;
-
- let subquery = Arc::new(
- LogicalPlanBuilder::from(inner_scan)
- .filter(
- out_ref_col(DataType::Int32, "outer_t.grp").lt(col("inner_t.grp")),
- )?
- .project(vec![col("inner_t.id")])?
- .build()?,
- );
-
- let plan = LogicalPlanBuilder::from(outer_scan)
- .filter(not_in_subquery(col("outer_t.id"), subquery).is_null())?
- .build()?;
-
- let optimized = optimize_with_decorrelate(plan)?;
- assert!(
- has_non_null_aware_left_mark_join(&optimized),
- "{}",
- optimized.display_indent_schema()
- );
-
- Ok(())
- }
-
#[test]
fn in_subquery_both_side_expr() -> Result<()> {
let table_scan = test_table_scan()?;
diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
index 050f223f9b..33e6099900 100644
--- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
+++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
@@ -655,11 +655,15 @@ CREATE TABLE naconst_corr_t2(id INT, g INT) AS VALUES (1, 1), (NULL, 2);
# A non-equality correlation stays in the join filter. Row (1, 1) sees the NULL
# of (NULL, 2), so its `NOT IN` is UNKNOWN; row (2, 2) sees an empty subquery,
# so its `NOT IN` is TRUE. Expected results verified with DuckDB.
-query error DataFusion error: Error during planning: null_aware LeftAnti join requires equi\-join keys, but the join has none
+query I
SELECT id FROM naconst_corr_t1 WHERE 3 NOT IN (SELECT id FROM naconst_corr_t2 WHERE naconst_corr_t2.g > naconst_corr_t1.g) ORDER BY id;
+----
+2
-query error DataFusion error: Error during planning: null_aware LeftAnti join requires equi\-join keys, but the join has none
+query I
SELECT id FROM naconst_corr_t1 WHERE NOT (3 IN (SELECT id FROM naconst_corr_t2 WHERE naconst_corr_t2.g > naconst_corr_t1.g)) ORDER BY id;
+----
+2
statement ok
DROP TABLE naconst_corr_t1;
@@ -683,17 +687,17 @@ query I
SELECT z FROM naconst_corr_t3 WHERE 1 NOT IN (SELECT naconst_corr_t4.id FROM naconst_corr_t4 WHERE naconst_corr_t4.z = naconst_corr_t3.z) ORDER BY z;
----
7
-10
+NULL
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
query I
SELECT z FROM naconst_corr_t3 WHERE 2 NOT IN (SELECT naconst_corr_t4.id FROM naconst_corr_t4 WHERE naconst_corr_t4.z = naconst_corr_t3.z) ORDER BY z;
----
-10
+NULL
-query error DataFusion error: Error during planning: null_aware LeftAnti join requires equi\-join keys, but the join has none
+query I
SELECT z FROM naconst_corr_t3 WHERE 1 NOT IN (SELECT naconst_corr_t4.id FROM naconst_corr_t4 WHERE naconst_corr_t4.z > naconst_corr_t3.z) ORDER BY z;
+----
+NULL
# The projected value column must be the FIRST equi-join key: a null-aware hash
# join reads `on[0]` as the `NOT IN` value key and `on[1..]` as correlation
@@ -702,16 +706,19 @@ query TT
EXPLAIN SELECT z FROM naconst_corr_t3 WHERE 1 NOT IN (SELECT naconst_corr_t4.id FROM naconst_corr_t4 WHERE naconst_corr_t4.z = naconst_corr_t3.z);
----
logical_plan
-01)LeftAnti Join: naconst_corr_t3.z = __correlated_sq_1.z Filter: Int64(1) = __correlated_sq_1.naconst_corr_t4.id null_aware
-02)--TableScan: naconst_corr_t3 projection=[z]
-03)--SubqueryAlias: __correlated_sq_1
-04)----Projection: CAST(naconst_corr_t4.id AS Int64), naconst_corr_t4.z
-05)------TableScan: naconst_corr_t4 projection=[id, z]
+01)Projection: naconst_corr_t3.z
+02)--LeftAnti Join: __correlated_sq_1_value = __correlated_sq_1.naconst_corr_t4.id, naconst_corr_t3.z = __correlated_sq_1.z null_aware
+03)----Projection: naconst_corr_t3.z, Int64(1) AS __correlated_sq_1_value
+04)------TableScan: naconst_corr_t3 projection=[z]
+05)----SubqueryAlias: __correlated_sq_1
+06)------Projection: CAST(naconst_corr_t4.id AS Int64), naconst_corr_t4.z
+07)--------TableScan: naconst_corr_t4 projection=[id, z]
physical_plan
-01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(z@0, z@1)], filter=1 = naconst_corr_t4.id@0, null_aware
-02)--DataSourceExec: partitions=1, partition_sizes=[1]
-03)--ProjectionExec: expr=[CAST(id@0 AS Int64) as naconst_corr_t4.id, z@1 as z]
-04)----DataSourceExec: partitions=1, partition_sizes=[1]
+01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(__correlated_sq_1_value@1, naconst_corr_t4.id@0), (z@0, z@1)], projection=[z@0], null_aware
+02)--ProjectionExec: expr=[z@0 as z, 1 as __correlated_sq_1_value]
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
+04)--ProjectionExec: expr=[CAST(id@0 AS Int64) as naconst_corr_t4.id, z@1 as z]
+05)----DataSourceExec: partitions=1, partition_sizes=[1]
statement ok
DROP TABLE naconst_corr_t3;
diff --git a/datafusion/sqllogictest/test_files/null_aware_mark_join.slt b/datafusion/sqllogictest/test_files/null_aware_mark_join.slt
index e4e14053c8..c45d87b51e 100644
--- a/datafusion/sqllogictest/test_files/null_aware_mark_join.slt
+++ b/datafusion/sqllogictest/test_files/null_aware_mark_join.slt
@@ -663,23 +663,19 @@ CREATE TABLE nam_res_inner(id INT, z INT, g INT) AS VALUES
(4, 35, 2),
(NULL, 2, 2);
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
query II rowsort
SELECT id, z FROM nam_res_outer
WHERE (id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.z < nam_res_outer.z)) IS NULL;
----
+2 20
+NULL 30
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
query II rowsort
SELECT id, z FROM nam_res_outer
WHERE (id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.z < nam_res_outer.z)) IS TRUE;
----
-2 20
5 1
NULL 1
-NULL 30
query II rowsort
SELECT id, z FROM nam_res_outer
@@ -689,36 +685,33 @@ WHERE (id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.z < nam_res_outer.z))
4 40
# `NOT mark` must stay NULL for UNKNOWN rows instead of turning into TRUE.
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
query II rowsort
SELECT id, z FROM nam_res_outer
WHERE NOT (id IN (SELECT i.id FROM nam_res_inner i WHERE i.z < nam_res_outer.z)) OR id = 4;
----
-2 20
4 40
5 1
NULL 1
-NULL 30
# Positive IN goes through the same mark.
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
query II rowsort
SELECT id, z FROM nam_res_outer
WHERE (id IN (SELECT i.id FROM nam_res_inner i WHERE i.z > nam_res_outer.z)) IS NULL;
----
+1 10
+2 20
+4 40
+5 1
+NULL 1
+NULL 30
# Equality and non-equality correlation together.
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
query II rowsort
SELECT id, z FROM nam_res_outer
WHERE (id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.g = nam_res_outer.g AND i.z < nam_res_outer.z)) IS NULL;
----
+NULL 30
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
query II rowsort
SELECT id, z FROM nam_res_outer
WHERE (id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.g = nam_res_outer.g AND i.z < nam_res_outer.z)) IS TRUE;
@@ -726,7 +719,6 @@ WHERE (id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.g = nam_res_outer.g A
2 20
5 1
NULL 1
-NULL 30
# Equality correlation only (already null-aware; control).
query II rowsort
@@ -774,8 +766,6 @@ physical_plan
# The negated form of the same query stays null-aware, because `NOT mark` tells
# NULL from FALSE.
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
query TT
EXPLAIN SELECT id FROM nam_res_outer o
WHERE o.z > 35 OR o.id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.z < o.z);
@@ -783,7 +773,7 @@ WHERE o.z > 35 OR o.id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.z < o.z)
logical_plan
01)Projection: o.id
02)--Filter: o.z > Int32(35) OR NOT __correlated_sq_1.mark
-03)----LeftMark Join: o.id = __correlated_sq_1.id Filter: __correlated_sq_1.z < o.z
+03)----LeftMark Join: o.id = __correlated_sq_1.id Filter: __correlated_sq_1.z < o.z null_aware
04)------SubqueryAlias: o
05)--------TableScan: nam_res_outer projection=[id, z]
06)------SubqueryAlias: __correlated_sq_1
@@ -792,7 +782,7 @@ logical_plan
physical_plan
01)FilterExec: z@1 > 35 OR NOT mark@2, projection=[id@0]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-03)----HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(id@0, id@0)], filter=z@1 < z@0
+03)----HashJoinExec: mode=CollectLeft, join_type=LeftMark, on=[(id@0, id@0)], filter=z@1 < z@0, null_aware
04)------DataSourceExec: partitions=1, partition_sizes=[1]
05)------DataSourceExec: partitions=1, partition_sizes=[1]
@@ -801,12 +791,12 @@ physical_plan
statement ok
SET datafusion.execution.batch_size = 1;
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
query II rowsort
SELECT id, z FROM nam_res_outer
WHERE (id NOT IN (SELECT i.id FROM nam_res_inner i WHERE i.z < nam_res_outer.z)) IS NULL;
----
+2 20
+NULL 30
statement ok
RESET datafusion.execution.batch_size;
@@ -925,8 +915,6 @@ CREATE TABLE nam_guard_inner(id INT, z INT) AS VALUES (1,5), (NULL,50);
# A non-negated `IN` reached through `OR` keeps the same rows with a NULL mark
# and with a FALSE mark, so the join must NOT be null-aware. It is then free to
# swap sides to `RightMark` as well.
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
query TT
EXPLAIN SELECT id FROM nam_guard_outer
WHERE id IN (SELECT i.id FROM nam_guard_inner i) OR z > 25;
@@ -934,7 +922,7 @@ WHERE id IN (SELECT i.id FROM nam_guard_inner i) OR z > 25;
logical_plan
01)Projection: nam_guard_outer.id
02)--Filter: __correlated_sq_1.mark OR nam_guard_outer.z > Int32(25)
-03)----LeftMark Join: nam_guard_outer.id = __correlated_sq_1.id null_aware
+03)----LeftMark Join: nam_guard_outer.id = __correlated_sq_1.id
04)------TableScan: nam_guard_outer projection=[id, z]
05)------SubqueryAlias: __correlated_sq_1
06)--------SubqueryAlias: i
@@ -942,7 +930,7 @@ logical_plan
physical_plan
01)FilterExec: mark@2 OR z@1 > 25, projection=[id@0]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-03)----HashJoinExec: mode=CollectLeft, join_type=LeftMark, on=[(id@0, id@0)], null_aware
+03)----HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(id@0, id@0)]
04)------DataSourceExec: partitions=1, partition_sizes=[1]
05)------DataSourceExec: partitions=1, partition_sizes=[1]
@@ -950,8 +938,6 @@ physical_plan
# two-valued, so its mark is never NULL and it cannot make the `IN` mark
# observable. Treating it as negative cost this query the plain join and the
# `RightMark` swap.
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
query TT
EXPLAIN SELECT id FROM nam_guard_outer
WHERE id IN (SELECT i.id FROM nam_guard_inner i)
@@ -962,7 +948,7 @@ logical_plan
02)--Filter: __correlated_sq_1.mark OR NOT __correlated_sq_2.mark
03)----Projection: nam_guard_outer.id, __correlated_sq_1.mark, __correlated_sq_2.mark
04)------LeftMark Join: Filter: __correlated_sq_2.z > nam_guard_outer.z
-05)--------LeftMark Join: nam_guard_outer.id = __correlated_sq_1.id null_aware
+05)--------LeftMark Join: nam_guard_outer.id = __correlated_sq_1.id
06)----------TableScan: nam_guard_outer projection=[id, z]
07)----------SubqueryAlias: __correlated_sq_1
08)------------SubqueryAlias: i
@@ -975,15 +961,13 @@ physical_plan
02)--NestedLoopJoinExec: join_type=RightMark, filter=z@1 > z@0, projection=[id@0, mark@2, mark@3]
03)----DataSourceExec: partitions=1, partition_sizes=[1]
04)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-05)------HashJoinExec: mode=CollectLeft, join_type=LeftMark, on=[(id@0, id@0)], null_aware
+05)------HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(id@0, id@0)]
06)--------DataSourceExec: partitions=1, partition_sizes=[1]
07)--------DataSourceExec: partitions=1, partition_sizes=[1]
# A constant value expression is projected as an outer column only when the
# join ends up null-aware. This mark join does not, so the constant stays a
# filter and no `__correlated_sq_*_value` column is produced.
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
query TT
EXPLAIN SELECT id FROM nam_guard_outer
WHERE 3 IN (SELECT i.id FROM nam_guard_inner i) OR z > 25;
@@ -991,22 +975,23 @@ WHERE 3 IN (SELECT i.id FROM nam_guard_inner i) OR z > 25;
logical_plan
01)Projection: nam_guard_outer.id
02)--Filter: __correlated_sq_1.mark OR nam_guard_outer.z > Int32(25)
-03)----Projection: nam_guard_outer.id, nam_guard_outer.z, __correlated_sq_1.mark
-04)------LeftMark Join: __correlated_sq_1_value = __correlated_sq_1.i.id null_aware
-05)--------Projection: nam_guard_outer.id, nam_guard_outer.z, Int64(3) AS __correlated_sq_1_value
-06)----------TableScan: nam_guard_outer projection=[id, z]
-07)--------SubqueryAlias: __correlated_sq_1
-08)----------Projection: CAST(i.id AS Int64)
-09)------------SubqueryAlias: i
-10)--------------TableScan: nam_guard_inner projection=[id]
+03)----LeftMark Join:
+04)------TableScan: nam_guard_outer projection=[id, z]
+05)------SubqueryAlias: __correlated_sq_1
+06)--------Projection: CAST(i.id AS Int64)
+07)----------SubqueryAlias: i
+08)------------Filter: nam_guard_inner.id = Int32(3)
+09)--------------TableScan: nam_guard_inner projection=[id]
physical_plan
01)FilterExec: mark@2 OR z@1 > 25, projection=[id@0]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-03)----HashJoinExec: mode=CollectLeft, join_type=LeftMark, on=[(__correlated_sq_1_value@2, i.id@0)], projection=[id@0, z@1, mark@3], null_aware
-04)------ProjectionExec: expr=[id@0 as id, z@1 as z, 3 as __correlated_sq_1_value]
-05)--------DataSourceExec: partitions=1, partition_sizes=[1]
-06)------ProjectionExec: expr=[CAST(id@0 AS Int64) as i.id]
-07)--------DataSourceExec: partitions=1, partition_sizes=[1]
+03)----NestedLoopJoinExec: join_type=RightMark
+04)------CoalescePartitionsExec
+05)--------ProjectionExec: expr=[CAST(id@0 AS Int64) as i.id]
+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+07)------------FilterExec: id@0 = 3
+08)--------------DataSourceExec: partitions=1, partition_sizes=[1]
+09)------DataSourceExec: partitions=1, partition_sizes=[1]
statement ok
DROP TABLE nam_guard_outer;
diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt
index ca0b0b1ff8..c208d7ab19 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -730,8 +730,6 @@ logical_plan
06)--EmptyRelation: rows=1
#simple_uncorrelated_scalar_subquery2
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
query TT
explain select (select count(*) from t1) as b, (select count(1) from t2)
----
@@ -1254,8 +1252,6 @@ where t1.t1_id > 40 or t1.t1_id in (select t2.t2_id from t2 where t1.t1_int > 0)
44 d 4
# not_in_subquery_to_join_with_correlated_outer_filter_disjunction
-# Pinned to today's behaviour, which is incorrect. See
-# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
query TT
explain select t1.t1_id,
t1.t1_name,
@@ -1266,7 +1262,7 @@ where t1.t1_id = 11 or t1.t1_id + 12 not in (select t2.t2_id + 1 from t2 where t
logical_plan
01)Projection: t1.t1_id, t1.t1_name, t1.t1_int
02)--Filter: t1.t1_id = Int32(11) OR NOT __correlated_sq_1.mark
-03)----LeftMark Join: CAST(t1.t1_id AS Int64) + Int64(12) = __correlated_sq_1.t2.t2_id + Int64(1) Filter: t1.t1_int > Int32(0)
+03)----LeftMark Join: CAST(t1.t1_id AS Int64) + Int64(12) = __correlated_sq_1.t2.t2_id + Int64(1) Filter: t1.t1_int > Int32(0) null_aware
04)------TableScan: t1 projection=[t1_id, t1_name, t1_int]
05)------SubqueryAlias: __correlated_sq_1
06)--------Projection: CAST(t2.t2_id AS Int64) + Int64(1)