Apache DataFusion SQL Query Engine
fix: handle `IS TRUE` correctly in `EliminateOuterJoin` (#22444)
Neil Conway50013e556911
diff --git a/datafusion/optimizer/src/eliminate_outer_join.rs b/datafusion/optimizer/src/eliminate_outer_join.rsindex cd060469b2..748b04d5cf 100644--- a/datafusion/optimizer/src/eliminate_outer_join.rs+++ b/datafusion/optimizer/src/eliminate_outer_join.rs@@ -80,11 +80,11 @@ impl OptimizerRule for EliminateOuterJoin {match plan {LogicalPlan::Filter(mut filter) => match Arc::unwrap_or_clone(filter.input) {LogicalPlan::Join(join) => {- let mut non_nullable_cols: Vec<Column> = vec![];+ let mut null_rejecting_cols: Vec<Column> = vec![];- extract_non_nullable_columns(+ extract_null_rejecting_columns(&filter.predicate,- &mut non_nullable_cols,+ &mut null_rejecting_cols,join.left.schema(),join.right.schema(),true,@@ -93,7 +93,7 @@ impl OptimizerRule for EliminateOuterJoin {let new_join_type = if join.join_type.is_outer() {let mut left_non_nullable = false;let mut right_non_nullable = false;- for col in non_nullable_cols.iter() {+ for col in null_rejecting_cols.iter() {if join.left.schema().has_column(col) {left_non_nullable = true;}@@ -163,62 +163,49 @@ pub fn eliminate_outer(new_join_type}-/// Recursively traverses expr, if expr returns false when-/// any inputs are null, treats columns of both sides as non_nullable columns.+/// Find the columns that `expr` rejects NULL on. If any of these columns are+/// NULL, `expr` is guaranteed to evaluate to NULL or false, and the row+/// therefore cannot survive a WHERE clause. Matching columns are appended to+/// `null_rejecting_cols`.///-/// For and/or expr, extracts from all sub exprs and merges the columns.-/// For or expr, if one of sub exprs returns true, discards all columns from or expr.-/// For IS NOT NULL/NOT expr, always returns false for NULL input.-/// extracts columns from these exprs.-/// For all other exprs, fall through-fn extract_non_nullable_columns(+/// The caller uses the result to decide whether an outer join's null-padded+/// rows could survive the predicate above the join: if a column from the+/// nullable side appears in `null_rejecting_cols`, it cannot, and the outer+/// join can be converted to an inner join.+///+/// `left_schema` and `right_schema` are the join's two child schemas.+/// `top_level` is true at the root of the WHERE predicate and false on each+/// recursion.+fn extract_null_rejecting_columns(expr: &Expr,- non_nullable_cols: &mut Vec<Column>,+ null_rejecting_cols: &mut Vec<Column>,left_schema: &Arc<DFSchema>,right_schema: &Arc<DFSchema>,top_level: bool,) {match expr {Expr::Column(col) => {- non_nullable_cols.push(col.clone());+ null_rejecting_cols.push(col.clone());}Expr::BinaryExpr(BinaryExpr { left, op, right }) => match op {- // If one of the inputs are null for these operators, the results should be false.- Operator::Eq- | Operator::NotEq- | Operator::Lt- | Operator::LtEq- | Operator::Gt- | Operator::GtEq => {- extract_non_nullable_columns(- left,- non_nullable_cols,- left_schema,- right_schema,- false,- );- extract_non_nullable_columns(- right,- non_nullable_cols,- left_schema,- right_schema,- false,- )- }Operator::And | Operator::Or => {- // treat And as Or if does not from top level, such as- // not (c1 < 10 and c2 > 100)+ // AND distributes only down a top-level AND chain in the WHERE+ // clause: each conjunct is independently null- rejecting, so+ // any column either side discovers is a column the WHERE+ // rejects NULL on. Once an AND appears below any other context,+ // we fall back to the per-side analysis used for OR, because+ // the context might influence whether the row is filtered.if top_level && *op == Operator::And {- extract_non_nullable_columns(+ extract_null_rejecting_columns(left,- non_nullable_cols,+ null_rejecting_cols,left_schema,right_schema,top_level,);- extract_non_nullable_columns(+ extract_null_rejecting_columns(right,- non_nullable_cols,+ null_rejecting_cols,left_schema,right_schema,top_level,@@ -226,124 +213,139 @@ fn extract_non_nullable_columns(return;}- let mut left_non_nullable_cols: Vec<Column> = vec![];- let mut right_non_nullable_cols: Vec<Column> = vec![];-- extract_non_nullable_columns(+ // OR (and nested AND): a row survives if EITHER operand returns+ // true. We can credit a join side as null-rejecting only when+ // BOTH operands independently reject NULL on a column from that+ // side — otherwise the other branch could let the NULL row+ // through.+ let mut left_cols: Vec<Column> = vec![];+ let mut right_cols: Vec<Column> = vec![];+ extract_null_rejecting_columns(left,- &mut left_non_nullable_cols,+ &mut left_cols,left_schema,right_schema,top_level,);- extract_non_nullable_columns(+ extract_null_rejecting_columns(right,- &mut right_non_nullable_cols,+ &mut right_cols,left_schema,right_schema,top_level,);- // for query: select *** from a left join b where b.c1 ... or b.c2 ...- // this can be eliminated to inner join.- // for query: select *** from a left join b where a.c1 ... or b.c2 ...- // this can not be eliminated.- // If columns of relation exist in both sub exprs, any columns of this relation- // can be added to non nullable columns.- if !left_non_nullable_cols.is_empty()- && !right_non_nullable_cols.is_empty()- {- for left_col in &left_non_nullable_cols {- for right_col in &right_non_nullable_cols {- if (left_schema.has_column(left_col)- && left_schema.has_column(right_col))- || (right_schema.has_column(left_col)- && right_schema.has_column(right_col))- {- non_nullable_cols.push(left_col.clone());- break;- }- }+ let find_on = |cols: &[Column], schema: &DFSchema| {+ cols.iter().find(|c| schema.has_column(c)).cloned()+ };+ for schema in [left_schema, right_schema] {+ if let (Some(c), Some(_)) =+ (find_on(&left_cols, schema), find_on(&right_cols, schema))+ {+ null_rejecting_cols.push(c);}}}+ // Any other operator that DataFusion declares as NULL-on-NULL:+ // recurse into both operands so we collect their columns.+ op if op.returns_null_on_null() => {+ extract_null_rejecting_columns(+ left,+ null_rejecting_cols,+ left_schema,+ right_schema,+ false,+ );+ extract_null_rejecting_columns(+ right,+ null_rejecting_cols,+ left_schema,+ right_schema,+ false,+ )+ }+ // All other operators (notably including IS [ NOT ] DISTINCT FROM)+ // are declared as not null-propagating, so they don't contribute+ // any null-rejecting columns._ => {}},- Expr::Not(arg) => extract_non_nullable_columns(+ Expr::Not(arg) | Expr::Negative(arg) => extract_null_rejecting_columns(arg,- non_nullable_cols,+ null_rejecting_cols,left_schema,right_schema,false,),- Expr::IsNotNull(arg) => {+ // IS NOT NULL / IS TRUE / IS FALSE / IS NOT UNKNOWN all return FALSE on+ // NULL input. At the top of a WHERE clause, that FALSE filters the row+ // and so we can recurse; below the top level the surrounding context+ // may transform that FALSE into something that accepts NULL rows,+ // making the recursion unsound.+ Expr::IsNotNull(arg)+ | Expr::IsTrue(arg)+ | Expr::IsFalse(arg)+ | Expr::IsNotUnknown(arg) => {if !top_level {return;}- extract_non_nullable_columns(+ extract_null_rejecting_columns(arg,- non_nullable_cols,+ null_rejecting_cols,left_schema,right_schema,false,)}Expr::Cast(Cast { expr, field: _ })- | Expr::TryCast(TryCast { expr, field: _ }) => extract_non_nullable_columns(+ | Expr::TryCast(TryCast { expr, field: _ }) => extract_null_rejecting_columns(expr,- non_nullable_cols,+ null_rejecting_cols,left_schema,right_schema,false,),// IN list and BETWEEN are null-rejecting on the input expression:- // if the input column is NULL, the result is NULL (filtered out),- // regardless of whether the list/range contains NULLs.- Expr::InList(InList { expr, .. }) => extract_non_nullable_columns(+ // NULL input yields a NULL result, regardless of whether the list+ // or range bounds themselves contain NULLs.+ Expr::InList(InList { expr, .. }) => extract_null_rejecting_columns(expr,- non_nullable_cols,+ null_rejecting_cols,left_schema,right_schema,false,),- Expr::Between(between) => extract_non_nullable_columns(+ Expr::Between(between) => extract_null_rejecting_columns(&between.expr,- non_nullable_cols,+ null_rejecting_cols,left_schema,right_schema,false,),- // LIKE is null-rejecting: if either the input column or the pattern- // is NULL, the result is NULL (filtered out by WHERE).Expr::Like(Like { expr, pattern, .. }) => {- extract_non_nullable_columns(+ extract_null_rejecting_columns(expr,- non_nullable_cols,+ null_rejecting_cols,left_schema,right_schema,false,);- extract_non_nullable_columns(+ extract_null_rejecting_columns(pattern,- non_nullable_cols,+ null_rejecting_cols,left_schema,right_schema,false,);}- // IS TRUE, IS FALSE, and IS NOT UNKNOWN are null-rejecting:- // if the input is NULL, they return false (filtered out by WHERE).- // Note: IS NOT TRUE, IS NOT FALSE, and IS UNKNOWN are NOT null-rejecting- // because they return true for NULL input.- Expr::IsTrue(arg) | Expr::IsFalse(arg) | Expr::IsNotUnknown(arg) => {- extract_non_nullable_columns(- arg,- non_nullable_cols,- left_schema,- right_schema,- false,- )- }+ // Anything not handled above contributes no null-rejecting+ // columns. Two categories worth calling out:+ // - IS NULL, IS NOT TRUE, IS NOT FALSE, IS UNKNOWN — return+ // TRUE on NULL input, so they actively *accept* NULL rows+ // and are intentionally excluded.+ // - Function calls (scalar / aggregate / window / UDF),+ // scalar subqueries, struct/list accessors, aliases,+ // literals, etc. — we don't have a uniform NULL-propagation+ // guarantee for these cases, so we conservatively skip them._ => {}}}@@ -360,7 +362,7 @@ mod tests {Operator::{And, Or},binary_expr, cast, col, lit,logical_plan::builder::LogicalPlanBuilder,- try_cast,+ not, try_cast,};macro_rules! assert_optimized_plan_equal {@@ -896,6 +898,83 @@ mod tests {")}+ #[test]+ fn no_eliminate_left_with_not_is_true() -> Result<()> {+ let t1 = test_table_scan_with_name("t1")?;+ let t2 = test_table_scan_with_name("t2")?;++ // NOT(<x> IS TRUE) is equivalent to (<x> IS NOT TRUE): TRUE when+ // <x> is FALSE OR NULL. So `WHERE NOT((t2.b > 5) IS TRUE)` accepts+ // rows where t2.b is NULL (because t2.b > 5 is NULL → IS TRUE is+ // FALSE → NOT FALSE = TRUE). The LEFT JOIN must NOT be converted.+ let plan = LogicalPlanBuilder::from(t1)+ .join(+ t2,+ JoinType::Left,+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),+ None,+ )?+ .filter(not(col("t2.b").gt(lit(5u32)).is_true()))?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Filter: NOT t2.b > UInt32(5) IS TRUE+ Left Join: t1.a = t2.a+ TableScan: t1+ TableScan: t2+ ")+ }++ #[test]+ fn no_eliminate_left_with_not_is_false() -> Result<()> {+ let t1 = test_table_scan_with_name("t1")?;+ let t2 = test_table_scan_with_name("t2")?;++ // Same shape, IS FALSE: NOT(<x> IS FALSE) accepts NULL on the+ // inner column.+ let plan = LogicalPlanBuilder::from(t1)+ .join(+ t2,+ JoinType::Left,+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),+ None,+ )?+ .filter(not(col("t2.b").gt(lit(5u32)).is_false()))?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Filter: NOT t2.b > UInt32(5) IS FALSE+ Left Join: t1.a = t2.a+ TableScan: t1+ TableScan: t2+ ")+ }++ #[test]+ fn no_eliminate_left_with_not_is_not_unknown() -> Result<()> {+ let t1 = test_table_scan_with_name("t1")?;+ let t2 = test_table_scan_with_name("t2")?;++ // Same shape, IS NOT UNKNOWN: NOT(<x> IS NOT UNKNOWN) is+ // equivalent to (<x> IS UNKNOWN), which is TRUE when <x> is NULL.+ let plan = LogicalPlanBuilder::from(t1)+ .join(+ t2,+ JoinType::Left,+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),+ None,+ )?+ .filter(not(col("t2.b").gt(lit(5u32)).is_not_unknown()))?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Filter: NOT t2.b > UInt32(5) IS NOT UNKNOWN+ Left Join: t1.a = t2.a+ TableScan: t1+ TableScan: t2+ ")+ }+#[test]fn eliminate_full_with_type_cast() -> Result<()> {let t1 = test_table_scan_with_name("t1")?;@@ -1171,4 +1250,122 @@ mod tests {TableScan: t2")}++ #[test]+ fn eliminate_left_with_arithmetic_predicate() -> Result<()> {+ let t1 = test_table_scan_with_name("t1")?;+ let t2 = test_table_scan_with_name("t2")?;++ // t2.b * 2 + 1 > 10 is null-rejecting on t2.b: arithmetic+ // operators propagate NULL, so the whole expression is NULL when+ // t2.b is NULL, and NULL > 10 is filtered out by WHERE.+ let plan = LogicalPlanBuilder::from(t1)+ .join(+ t2,+ JoinType::Left,+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),+ None,+ )?+ .filter(+ binary_expr(+ binary_expr(col("t2.b"), Operator::Multiply, lit(2u32)),+ Operator::Plus,+ lit(1u32),+ )+ .gt(lit(10u32)),+ )?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Filter: t2.b * UInt32(2) + UInt32(1) > UInt32(10)+ Inner Join: t1.a = t2.a+ TableScan: t1+ TableScan: t2+ ")+ }++ #[test]+ fn eliminate_left_with_negative_predicate() -> Result<()> {+ let t1 = test_table_scan_with_name("t1")?;+ let t2 = test_table_scan_with_name("t2")?;++ // Unary minus propagates NULL: -NULL is NULL, so `WHERE -t2.b > 0`+ // is null-rejecting on t2.b.+ let plan = LogicalPlanBuilder::from(t1)+ .join(+ t2,+ JoinType::Left,+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),+ None,+ )?+ .filter(Expr::Negative(Box::new(col("t2.b"))).gt(lit(0u32)))?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Filter: (- t2.b) > UInt32(0)+ Inner Join: t1.a = t2.a+ TableScan: t1+ TableScan: t2+ ")+ }++ #[test]+ fn no_eliminate_left_with_is_distinct_from() -> Result<()> {+ let t1 = test_table_scan_with_name("t1")?;+ let t2 = test_table_scan_with_name("t2")?;++ // IS DISTINCT FROM is NOT null-rejecting: t2.b IS DISTINCT FROM 5 is+ // true when t2.b is NULL (NULL is distinct from 5). Padding rows from+ // a LEFT JOIN would survive the filter, so the LEFT JOIN must stay.+ let plan = LogicalPlanBuilder::from(t1)+ .join(+ t2,+ JoinType::Left,+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),+ None,+ )?+ .filter(binary_expr(+ col("t2.b"),+ Operator::IsDistinctFrom,+ lit(5u32),+ ))?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Filter: t2.b IS DISTINCT FROM UInt32(5)+ Left Join: t1.a = t2.a+ TableScan: t1+ TableScan: t2+ ")+ }++ #[test]+ fn no_eliminate_left_with_is_not_distinct_from() -> Result<()> {+ let t1 = test_table_scan_with_name("t1")?;+ let t2 = test_table_scan_with_name("t2")?;++ // IS NOT DISTINCT FROM is also not null-rejecting: t2.b IS NOT+ // DISTINCT FROM NULL is true when t2.b is NULL. The LEFT JOIN must+ // stay.+ let plan = LogicalPlanBuilder::from(t1)+ .join(+ t2,+ JoinType::Left,+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),+ None,+ )?+ .filter(binary_expr(+ col("t2.b"),+ Operator::IsNotDistinctFrom,+ lit(ScalarValue::UInt32(None)),+ ))?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Filter: t2.b IS NOT DISTINCT FROM UInt32(NULL)+ Left Join: t1.a = t2.a+ TableScan: t1+ TableScan: t2+ ")+ }}diff --git a/datafusion/sqllogictest/test_files/eliminate_outer_join.slt b/datafusion/sqllogictest/test_files/eliminate_outer_join.sltindex dff7692a44..d22a7f2e3c 100644--- a/datafusion/sqllogictest/test_files/eliminate_outer_join.slt+++ b/datafusion/sqllogictest/test_files/eliminate_outer_join.slt@@ -370,6 +370,66 @@ select * from t1 left join t2 on t1.a = t2.x where (t2.y > 150) is unknown;3 30 c NULL NULL NULLNULL 40 d NULL NULL NULL+# LEFT JOIN + WHERE NOT ((t2.y > 150) IS TRUE) -> stays LEFT JOIN+query TT+explain select * from t1 left join t2 on t1.a = t2.x where not ((t2.y > 150) is true);+----+logical_plan+01)Filter: NOT t2.y > Int32(150) IS TRUE+02)--Left Join: t1.a = t2.x+03)----TableScan: t1 projection=[a, b, c]+04)----TableScan: t2 projection=[x, y, z]++# Both the matched-with-low-y row AND the LEFT-padded NULL rows must+# survive.+query IITIIT rowsort+select * from t1 left join t2 on t1.a = t2.x where not ((t2.y > 150) is true);+----+1 10 a 1 100 p+3 30 c NULL NULL NULL+NULL 40 d NULL NULL NULL++# LEFT JOIN + WHERE NOT ((t2.y > 150) IS FALSE) -> stays LEFT JOIN+# NOT(<x> IS FALSE) is TRUE when <x> is TRUE OR NULL, so it accepts the+# LEFT-padded NULL rows and is not null-rejecting.+query TT+explain select * from t1 left join t2 on t1.a = t2.x where not ((t2.y > 150) is false);+----+logical_plan+01)Filter: NOT t2.y > Int32(150) IS FALSE+02)--Left Join: t1.a = t2.x+03)----TableScan: t1 projection=[a, b, c]+04)----TableScan: t2 projection=[x, y, z]++# The matched-with-high-y row AND the LEFT-padded NULL rows must survive.+query IITIIT rowsort+select * from t1 left join t2 on t1.a = t2.x where not ((t2.y > 150) is false);+----+2 20 b 2 200 q+3 30 c NULL NULL NULL+NULL 40 d NULL NULL NULL++# LEFT JOIN + WHERE NOT ((t2.y > 150) IS NOT UNKNOWN) -> stays LEFT JOIN+# NOT(<x> IS NOT UNKNOWN) is equivalent to IS UNKNOWN: TRUE only when+# <x> is NULL, so it accepts the LEFT-padded NULL rows and is not+# null-rejecting.+query TT+explain select * from t1 left join t2 on t1.a = t2.x where not ((t2.y > 150) is not unknown);+----+logical_plan+01)Filter: NOT t2.y > Int32(150) IS NOT UNKNOWN+02)--Left Join: t1.a = t2.x+03)----TableScan: t1 projection=[a, b, c]+04)----TableScan: t2 projection=[x, y, z]++# Only the LEFT-padded NULL rows (where t2.y > 150 evaluates to NULL)+# should survive.+query IITIIT rowsort+select * from t1 left join t2 on t1.a = t2.x where not ((t2.y > 150) is not unknown);+----+3 30 c NULL NULL NULL+NULL 40 d NULL NULL NULL+###### FULL JOIN → LEFT / RIGHT conversion tests###