Apache DataFusion SQL Query Engine
fix: plan aggregates over CASE conditions with multiple columns (#25684)
Stefan Wang5f2158269e24
diff --git a/datafusion/physical-expr/src/expressions/case.rs b/datafusion/physical-expr/src/expressions/case.rsindex c30eb9d332..439592e4df 100644--- a/datafusion/physical-expr/src/expressions/case.rs+++ b/datafusion/physical-expr/src/expressions/case.rs@@ -19,7 +19,7 @@ mod literal_lookup_table;use super::{Column, Literal};use crate::expressions::{- CastExpr, LambdaVariable, NegativeExpr, NotExpr, lit, try_cast,+ BinaryExpr, CastExpr, LambdaVariable, NegativeExpr, NotExpr, lit, try_cast,};use crate::{PhysicalExpr, ScalarFunctionExpr};use arrow::array::*;@@ -35,6 +35,8 @@ use datafusion_common::{internal_datafusion_err, internal_err,};use datafusion_expr::ColumnarValue;+use datafusion_expr_common::interval_arithmetic::NullableInterval;+use datafusion_expr_common::operator::Operator;use indexmap::IndexMap;use std::borrow::Cow;use std::collections::BTreeSet;@@ -1292,19 +1294,14 @@ impl PhysicalExpr for CaseExpr {Ok(e) => e,};- // Try to const evaluate the modified `when` expression.- let predicate_result = match evaluate_predicate(&with_null) {+ let can_be_true = match evaluate_predicate(&with_null).and_then(|bounds| {+ bounds.contains_value(ScalarValue::Boolean(Some(true)))+ }) {Err(e) => return Some(Err(e)),Ok(b) => b,};- match predicate_result {- // Evaluation was inconclusive or true, so the 'then' expression is reachable- None | Some(true) => Some(Ok(())),- // Evaluation proves the branch will never be taken.- // The most common pattern for this is `WHEN x IS NOT NULL THEN x`.- Some(false) => None,- }+ if can_be_true { Some(Ok(())) } else { None }});if let Some(nullable_then) = nullable_then {@@ -1516,12 +1513,8 @@ impl CaseExpr {}}-/// Attempts to const evaluate the given `predicate`.-/// Returns:-/// - `Some(true)` if the predicate evaluates to a truthy value.-/// - `Some(false)` if the predicate evaluates to a falsy value.-/// - `None` if the predicate could not be evaluated.-fn evaluate_predicate(predicate: &Arc<dyn PhysicalExpr>) -> Result<Option<bool>> {+/// Uses three-valued bounds when input columns prevent constant evaluation.+fn evaluate_predicate(predicate: &Arc<dyn PhysicalExpr>) -> Result<NullableInterval> {// Create a dummy record with no columns and one rowlet batch = RecordBatch::try_new_with_options(Arc::new(Schema::empty()),@@ -1531,15 +1524,24 @@ fn evaluate_predicate(predicate: &Arc<dyn PhysicalExpr>) -> Result<Option<bool>>// Evaluate the predicate and interpret the result as a booleanlet result = match predicate.evaluate(&batch) {- // An error during evaluation means we couldn't const evaluate the predicate, so return `None`- Err(_) => None,- Ok(ColumnarValue::Array(array)) => Some(+ Err(_) => {+ if let Some(expr) = predicate.downcast_ref::<BinaryExpr>()+ && matches!(expr.op(), Operator::And | Operator::Or)+ {+ return evaluate_predicate(expr.left())?+ .apply_operator(expr.op(), &evaluate_predicate(expr.right())?);+ }+ if let Some(expr) = predicate.downcast_ref::<NotExpr>() {+ return evaluate_predicate(expr.arg())?.not();+ }+ return Ok(NullableInterval::ANY_TRUTH_VALUE);+ }+ Ok(ColumnarValue::Array(array)) => {ScalarValue::try_from_array(array.as_ref(), 0)?- .cast_to(&DataType::Boolean)?,- ),- Ok(ColumnarValue::Scalar(scalar)) => Some(scalar.cast_to(&DataType::Boolean)?),+ }+ Ok(ColumnarValue::Scalar(scalar)) => scalar,};- Ok(result.map(|v| matches!(v, ScalarValue::Boolean(Some(true)))))+ Ok(result.cast_to(&DataType::Boolean)?.into())}fn replace_with_null(@@ -1594,7 +1596,7 @@ mod tests {use super::*;use crate::expressions;- use crate::expressions::{BinaryExpr, binary, cast, col, is_not_null};+ use crate::expressions::{binary, cast, col, is_not_null};use arrow::buffer::Buffer;use arrow::datatypes::DataType::Float64;use arrow::datatypes::Field;@@ -1603,7 +1605,6 @@ mod tests {use datafusion_common::plan_err;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_expr::type_coercion::binary::type_union_coercion;- use datafusion_expr_common::operator::Operator;use datafusion_physical_expr_common::physical_expr::fmt_sql;use half::f16;@@ -2796,6 +2797,145 @@ mod tests {case_expression_nullability(false)}+ #[rstest::rstest]+ #[case::and(Operator::And, false, false)]+ #[case::or(Operator::Or, false, true)]+ #[case::not_and(Operator::And, true, true)]+ #[case::not_or(Operator::Or, true, false)]+ fn test_case_expression_nullability_with_independent_columns(+ #[case] op: Operator,+ #[case] negated: bool,+ #[case] nullable: bool,+ #[values(false, true)] reverse: bool,+ ) -> Result<()> {+ let schema = Schema::new(vec![+ Field::new("a", DataType::Int32, true),+ Field::new("b", DataType::Int32, true),+ ]);+ let a: Arc<dyn PhysicalExpr> = col("a", &schema)?;+ let left: Arc<dyn PhysicalExpr> =+ binary(Arc::clone(&a), Operator::Gt, lit(1), &schema)?;+ let right: Arc<dyn PhysicalExpr> =+ binary(col("b", &schema)?, Operator::Lt, lit(3), &schema)?;+ let predicate: Arc<dyn PhysicalExpr> = if reverse {+ binary(right, op, left, &schema)?+ } else {+ binary(left, op, right, &schema)?+ };+ let predicate: Arc<dyn PhysicalExpr> = if negated {+ expressions::not(predicate)?+ } else {+ predicate+ };++ assert_nullability(when_then_else(&predicate, &a, &lit(0))?, &schema, nullable);+ Ok(())+ }++ #[rstest::rstest]+ #[case::and_over_or(false, false)]+ #[case::not_and_over_or(true, true)]+ fn test_case_expression_nullability_with_nested_predicates(+ #[case] negated: bool,+ #[case] nullable: bool,+ #[values(false, true)] reverse: bool,+ ) -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, true),+ Field::new("b", DataType::Int32, true),+ ]));+ let a: Arc<dyn PhysicalExpr> = col("a", &schema)?;+ let b: Arc<dyn PhysicalExpr> = col("b", &schema)?;++ // a > 1 AND (b < 3 OR b IS NULL)+ let a_gt_1: Arc<dyn PhysicalExpr> =+ binary(Arc::clone(&a), Operator::Gt, lit(1), &schema)?;+ let b_lt_3_or_null: Arc<dyn PhysicalExpr> = binary(+ binary(Arc::clone(&b), Operator::Lt, lit(3), &schema)?,+ Operator::Or,+ expressions::is_null(b)?,+ &schema,+ )?;+ let predicate: Arc<dyn PhysicalExpr> = if reverse {+ binary(b_lt_3_or_null, Operator::And, a_gt_1, &schema)?+ } else {+ binary(a_gt_1, Operator::And, b_lt_3_or_null, &schema)?+ };+ let predicate: Arc<dyn PhysicalExpr> = if negated {+ expressions::not(predicate)?+ } else {+ predicate+ };+ let expr = when_then_else(&predicate, &a, &lit(0))?;++ assert_nullability(Arc::clone(&expr), &schema, nullable);++ // Cross-check the inferred nullability against evaluation on every combination+ // of NULL and non-NULL inputs. With a = NULL and b = 3 the negated predicate is+ // TRUE, so the THEN branch yields NULL.+ let rows: Vec<(Option<i32>, Option<i32>)> = [None, Some(1), Some(2)]+ .into_iter()+ .flat_map(|a| [None, Some(2), Some(3)].into_iter().map(move |b| (a, b)))+ .collect();+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from_iter(rows.iter().map(|(a, _)| *a))),+ Arc::new(Int32Array::from_iter(rows.iter().map(|(_, b)| *b))),+ ],+ )?;+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ assert_eq!(result.null_count() > 0, nullable);+ Ok(())+ }++ #[rstest::rstest]+ #[case::literal(lit("invalid"))]+ #[case::and_left(Arc::new(BinaryExpr::new(+ lit("invalid"),+ Operator::And,+ make_col("flag", 0)+ )))]+ #[case::and_right(Arc::new(BinaryExpr::new(+ make_col("flag", 0),+ Operator::And,+ lit("invalid")+ )))]+ #[case::or_left(Arc::new(BinaryExpr::new(+ lit("invalid"),+ Operator::Or,+ make_col("flag", 0)+ )))]+ #[case::or_right(Arc::new(BinaryExpr::new(+ make_col("flag", 0),+ Operator::Or,+ lit("invalid")+ )))]+ #[case::not(Arc::new(NotExpr::new(lit("invalid"))))]+ fn test_case_expression_nullability_with_invalid_predicate(+ #[case] predicate: Arc<dyn PhysicalExpr>,+ ) -> Result<()> {+ let batch = RecordBatch::try_from_iter(vec![+ (+ "flag",+ Arc::new(BooleanArray::from(vec![Some(true), Some(false), None]))+ as ArrayRef,+ ),+ (+ "value",+ Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])) as ArrayRef,+ ),+ ])?;+ let schema = batch.schema();+ let expr = when_then_else(&predicate, &col("value", &schema)?, &lit(0))?;+ let error = expr+ .nullable(&schema)+ .expect_err("invalid predicates must fail nullability inference");+ assert!(error.to_string().contains("invalid"), "{error}");+ assert!(expr.evaluate(&batch).is_err());+ Ok(())+ }+fn case_expression_nullability(col_is_nullable: bool) -> Result<()> {let schema =Schema::new(vec![Field::new("foo", DataType::Int32, col_is_nullable)]);diff --git a/datafusion/sqllogictest/test_files/cse.slt b/datafusion/sqllogictest/test_files/cse.sltindex 6f8abd8085..5681c16d14 100644--- a/datafusion/sqllogictest/test_files/cse.slt+++ b/datafusion/sqllogictest/test_files/cse.slt@@ -214,6 +214,71 @@ physical_plan02)--ProjectionExec: expr=[a@0 = 1 as __common_expr_1, a@0 = 2 as __common_expr_2, a@0 + 3 as __common_expr_3, a@0 + 4 as __common_expr_4]03)----DataSourceExec: partitions=1, partition_sizes=[0]+statement ok+CREATE TABLE cse_nullable_case(a INT, b INT) AS VALUES (1, 1), (2, 2), (3, 3);++query I+SELECT sum(CASE WHEN (CASE WHEN a > 1 AND b < 3 THEN a ELSE 0 END)+ BETWEEN 1 AND 2 THEN 1 ELSE 0 END)+FROM cse_nullable_case;+----+1++query I+SELECT count(CASE WHEN (CASE WHEN a > 1 AND b < 3 THEN a ELSE 0 END)+ BETWEEN 1 AND 2 THEN 1 ELSE 0 END)+FROM cse_nullable_case;+----+3++query I+SELECT sum(CASE WHEN (CASE WHEN a > 1 AND b < 3 THEN a ELSE 0 END) >= 1+ AND (CASE WHEN a > 1 AND b < 3 THEN a ELSE 0 END) <= 2+ THEN 1 ELSE 0 END)+FROM cse_nullable_case;+----+1++statement ok+INSERT INTO cse_nullable_case VALUES (NULL, 2), (2, NULL), (NULL, NULL);++query II+SELECT sum(v), count(v)+FROM (SELECT CASE WHEN a > 1 AND b < 3 THEN a ELSE 0 END AS v FROM cse_nullable_case);+----+2 6++query II+SELECT sum(v), count(v)+FROM (SELECT CASE WHEN a > 1 OR b < 3 THEN a ELSE 0 END AS v FROM cse_nullable_case);+----+8 5++query II+SELECT sum(v), count(v)+FROM (SELECT CASE WHEN NOT (a > 1 OR b < 3) THEN a ELSE 0 END AS v FROM cse_nullable_case);+----+0 6++query II+SELECT sum(v), count(v)+FROM (SELECT CASE WHEN a > 1 AND b < 3 THEN a END AS v FROM cse_nullable_case);+----+2 1++query II+SELECT sum(v), count(v)+FROM (SELECT CASE WHEN a > 1 AND b < 3 THEN a ELSE b END AS v FROM cse_nullable_case);+----+8 4++query II+SELECT sum(v), count(v)+FROM (SELECT CASE WHEN a > 1 AND b < 3 THEN a ELSE 0 END AS v FROM cse_nullable_case)+WHERE false;+----+NULL 0+# Only conditionally evaluated expressionsquery TTEXPLAIN SELECT