Apache DataFusion SQL Query Engine
Fix physical distinctness expression nullability (#24244)
blinding-pixelsfdf7935a640e
diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rsindex 21c68089df..36b76f076d 100644--- a/datafusion/expr/src/expr_schema.rs+++ b/datafusion/expr/src/expr_schema.rs@@ -27,7 +27,9 @@ use crate::higher_order_function::HigherOrderReturnFieldArgs;use crate::type_coercion::functions::value_fields_with_higher_order_udf_and_lambdas;use crate::type_coercion::functions::{UDFCoercionExt, fields_with_udf};use crate::udf::ReturnFieldArgs;-use crate::{LogicalPlan, Projection, Subquery, WindowFunctionDefinition, utils};+use crate::{+ LogicalPlan, Operator, Projection, Subquery, WindowFunctionDefinition, utils,+};use arrow::compute::can_cast_types;use arrow::datatypes::FieldRef;use arrow::datatypes::{DataType, Field};@@ -390,9 +392,10 @@ impl ExprSchemable for Expr {Ok(expr_nullable | subquery_nullable)}Expr::ScalarSubquery(subquery) => Ok(scalar_subquery_nullable(subquery)),- Expr::BinaryExpr(BinaryExpr { left, right, .. }) => {- Ok(left.nullable(input_schema)? || right.nullable(input_schema)?)- }+ Expr::BinaryExpr(BinaryExpr { left, right, op }) => match op {+ Operator::IsDistinctFrom | Operator::IsNotDistinctFrom => Ok(false),+ _ => Ok(left.nullable(input_schema)? || right.nullable(input_schema)?),+ },Expr::Like(Like { expr, pattern, .. })| Expr::SimilarTo(Like { expr, pattern, .. }) => {Ok(expr.nullable(input_schema)? || pattern.nullable(input_schema)?)@@ -555,10 +558,14 @@ impl ExprSchemable for Expr {let mut coercer = BinaryTypeCoercer::new(lhs_type, op, rhs_type);coercer.set_lhs_spans(left.spans().cloned().unwrap_or_default());coercer.set_rhs_spans(right.spans().cloned().unwrap_or_default());+ let nullable = match op {+ Operator::IsDistinctFrom | Operator::IsNotDistinctFrom => false,+ _ => lhs_nullable || rhs_nullable,+ };Ok(Arc::new(Field::new(&schema_name,coercer.get_result_type()?,- lhs_nullable || rhs_nullable,+ nullable,)))}Expr::WindowFunction(window_function) => {@@ -899,12 +906,30 @@ mod tests {#[test]fn expr_schema_nullability() {- let expr = col("foo").eq(lit(1));- assert!(!expr.nullable(&MockExprSchema::new()).unwrap());- assert!(- expr.nullable(&MockExprSchema::new().with_nullable(true))- .unwrap()- );+ let cases = [+ (Operator::Eq, false, false),+ (Operator::Eq, true, true),+ (Operator::IsDistinctFrom, true, false),+ (Operator::IsNotDistinctFrom, true, false),+ ];++ for (op, input_nullable, expected) in cases {+ let expr = Expr::BinaryExpr(BinaryExpr::new(+ Box::new(col("foo")),+ op,+ Box::new(col("bar")),+ ));+ let schema = MockExprSchema::new()+ .with_data_type(DataType::Boolean)+ .with_nullable(input_nullable);++ assert_eq!(expr.nullable(&schema).unwrap(), expected, "{op}");+ assert_eq!(+ expr.to_field(&schema).unwrap().1.is_nullable(),+ expected,+ "{op}"+ );+ }test_is_expr_nullable!(is_null);test_is_expr_nullable!(is_not_null);diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rsindex c956b83178..dfb1d136d0 100644--- a/datafusion/physical-expr/src/expressions/binary.rs+++ b/datafusion/physical-expr/src/expressions/binary.rs@@ -530,7 +530,13 @@ impl PhysicalExpr for BinaryExpr {}fn nullable(&self, input_schema: &Schema) -> Result<bool> {- Ok(self.left.nullable(input_schema)? || self.right.nullable(input_schema)?)+ match self.op {+ Operator::IsDistinctFrom | Operator::IsNotDistinctFrom => Ok(false),+ _ => {+ Ok(self.left.nullable(input_schema)?+ || self.right.nullable(input_schema)?)+ }+ }}fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {@@ -4193,6 +4199,27 @@ mod tests {apply_logic_op(&schema, &a, &b, Operator::IsDistinctFrom, expected).unwrap();}+ #[test]+ fn distinct_from_op_nullability() -> Result<()> {+ let schema = Schema::new(vec![+ Field::new("nullable", DataType::Boolean, true),+ Field::new("non_nullable", DataType::Boolean, false),+ ]);+ let cases = [+ (Operator::IsDistinctFrom, "nullable", false),+ (Operator::IsNotDistinctFrom, "nullable", false),+ (Operator::Eq, "nullable", true),+ (Operator::Eq, "non_nullable", false),+ ];++ for (op, column, expected) in cases {+ let expr = BinaryExpr::new(col(column, &schema)?, op, lit(true));+ assert_eq!(expr.nullable(&schema)?, expected, "{op} with {column}");+ }++ Ok(())+ }+#[test]fn is_not_distinct_from_op_bool() {let (schema, a, b) = bool_test_arrays();diff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.sltindex 2bedb28a85..175607b620 100644--- a/datafusion/sqllogictest/test_files/scalar.slt+++ b/datafusion/sqllogictest/test_files/scalar.slt@@ -1864,6 +1864,26 @@ truefalsetrue+# Aggregate CSE preserves the non-nullability of truth predicates+# issue: https://github.com/apache/datafusion/issues/24096+query IIIIIII+SELECT+ SUM(CASE WHEN b IS TRUE THEN 1 ELSE 0 END),+ COUNT(CASE WHEN b IS TRUE THEN 1 END),+ SUM(CASE WHEN b IS FALSE THEN 1 ELSE 0 END),+ SUM(CASE WHEN b IS NOT TRUE THEN 1 ELSE 0 END),+ SUM(CASE WHEN b IS NOT FALSE THEN 1 ELSE 0 END),+ SUM(CASE WHEN b IS UNKNOWN THEN 1 ELSE 0 END),+ SUM(CASE WHEN b IS NOT UNKNOWN THEN 1 ELSE 0 END)+FROM (+ VALUES+ (TRUE),+ (FALSE),+ (CAST(NULL AS BOOLEAN))+) AS t(b);+----+1 1 1 2 2 1 2+# query_without_from()