Apache DataFusion SQL Query Engine
Fix CASE evaluation for custom column expressions (#24484)
Hasnaat hussain69a4c72264a3
diff --git a/datafusion/physical-expr/src/expressions/case.rs b/datafusion/physical-expr/src/expressions/case.rsindex e95e9e570c..dd98029c73 100644--- a/datafusion/physical-expr/src/expressions/case.rs+++ b/datafusion/physical-expr/src/expressions/case.rs@@ -18,10 +18,10 @@mod literal_lookup_table;use super::{Column, Literal};-use crate::PhysicalExpr;use crate::expressions::{CastExpr, LambdaVariable, NegativeExpr, NotExpr, lit, try_cast,};+use crate::{PhysicalExpr, ScalarFunctionExpr};use arrow::array::*;use arrow::compute::kernels::zip::zip;use arrow::compute::{@@ -133,6 +133,7 @@ impl CaseBody {// Determine the set of columns that are used in all the expressions of the case body.// Use an ordered set so lambda variables continue to be positioned after columnslet mut used_column_indices = BTreeSet::<usize>::new();+ let mut supports_projection = true;let mut collect_column_indices = |expr: &Arc<dyn PhysicalExpr>| {expr.apply(|expr| {if let Some(column) = expr.downcast_ref::<Column>() {@@ -141,6 +142,12 @@ impl CaseBody {expr.downcast_ref::<LambdaVariable>(){used_column_indices.insert(lambda_variable.index());+ } else if expr.downcast_ref::<Literal>().is_none()+ && expr.downcast_ref::<ScalarFunctionExpr>().is_none()+ && expr.children().is_empty()+ {+ // Unknown leaves may read input columns without exposing a Column child.+ supports_projection = false;}Ok(TreeNodeRecursion::Continue)})@@ -216,6 +223,7 @@ impl CaseBody {Ok(ProjectedCaseBody {projection,body: projected_body,+ supports_projection,})}}@@ -247,10 +255,32 @@ impl CaseBody {////// The projection vector and the rewritten expression (which only differs from the original in/// column reference indices) are held in a `ProjectedCaseBody`.+///+/// When `supports_projection` is false, evaluation uses the original body and full batch because+/// the expression may depend on input columns that are not visible in its children.#[derive(Debug, Hash, PartialEq, Eq)]struct ProjectedCaseBody {projection: Vec<usize>,body: CaseBody,+ /// Whether `body` is safe to evaluate against a projected batch.+ ///+ /// When false, `projection` and the rewritten `body` must not be used because an+ /// expression may read input columns without exposing them through its children.+ supports_projection: bool,+}++impl ProjectedCaseBody {+ /// Returns a projection when evaluating the derived body can avoid unused input columns.+ fn projection_for(&self, batch: &RecordBatch) -> Option<Vec<usize>> {+ let projection = self+ .projection+ .iter()+ .copied()+ .filter(|index| *index < batch.num_columns())+ .collect::<Vec<_>>();+ (self.supports_projection && projection.len() < batch.num_columns())+ .then_some(projection)+ }}/// The CASE expression is similar to a series of nested if/else and there are two forms that@@ -1049,14 +1079,7 @@ impl CaseExpr {projected: &ProjectedCaseBody,) -> Result<ColumnarValue> {let return_type = self.data_type(&batch.schema())?;- // projected.projection may include indexes of lambda variables not available on this batch- let projection = projected- .projection- .iter()- .copied()- .filter(|index| *index < batch.num_columns())- .collect::<Vec<_>>();- if projection.len() < batch.num_columns() {+ if let Some(projection) = projected.projection_for(batch) {let projected_batch = batch.project(&projection)?;projected.body@@ -1079,14 +1102,7 @@ impl CaseExpr {projected: &ProjectedCaseBody,) -> Result<ColumnarValue> {let return_type = self.data_type(&batch.schema())?;- // projected.projection may include indexes of lambda variables not available on this batch- let projection = projected- .projection- .iter()- .copied()- .filter(|index| *index < batch.num_columns())- .collect::<Vec<_>>();- if projection.len() < batch.num_columns() {+ if let Some(projection) = projected.projection_for(batch) {let projected_batch = batch.project(&projection)?;projected.body@@ -1204,23 +1220,14 @@ impl CaseExpr {)?))}}+ } else if let Some(projection) = projected.projection_for(batch) {+ // The case expressions do not use all the columns of the input batch.+ // Project first to reduce time spent filtering.+ let projected_batch = batch.project(&projection)?;+ projected.body.expr_or_expr(&projected_batch, when_value)} else {- // projected.projection may include indexes of lambda variables not available on this batch- let projection = projected- .projection- .iter()- .copied()- .filter(|index| *index < batch.num_columns())- .collect::<Vec<_>>();- if projection.len() < batch.num_columns() {- // The case expressions do not use all the columns of the input batch.- // Project first to reduce time spent filtering.- let projected_batch = batch.project(&projection)?;- projected.body.expr_or_expr(&projected_batch, when_value)- } else {- // All columns are used in the case expressions, so there is no need to project.- self.body.expr_or_expr(batch, when_value)- }+ // All columns are used in the case expressions, so there is no need to project.+ self.body.expr_or_expr(batch, when_value)}}@@ -1592,6 +1599,7 @@ mod tests {use arrow::datatypes::DataType::Float64;use arrow::datatypes::Field;use datafusion_common::cast::{as_float64_array, as_int32_array};+ use datafusion_common::config::ConfigOptions;use datafusion_common::plan_err;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_expr::type_coercion::binary::type_union_coercion;@@ -1599,6 +1607,55 @@ mod tests {use datafusion_physical_expr_common::physical_expr::fmt_sql;use half::f16;+ #[derive(Debug, Hash, PartialEq, Eq)]+ struct CustomColumn {+ inner: Column,+ }++ impl CustomColumn {+ fn new(name: &str, index: usize) -> Self {+ Self {+ inner: Column::new(name, index),+ }+ }+ }++ impl std::fmt::Display for CustomColumn {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ std::fmt::Display::fmt(&self.inner, f)+ }+ }++ impl PhysicalExpr for CustomColumn {+ fn data_type(&self, input_schema: &Schema) -> Result<DataType> {+ self.inner.data_type(input_schema)+ }++ fn nullable(&self, input_schema: &Schema) -> Result<bool> {+ self.inner.nullable(input_schema)+ }++ fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {+ self.inner.evaluate(batch)+ }++ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {+ vec![]+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn PhysicalExpr>>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ assert!(children.is_empty());+ Ok(self)+ }++ fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ self.inner.fmt_sql(f)+ }+ }+#[test]fn case_with_expr() -> Result<()> {let batch = case_test_batch()?;@@ -1894,6 +1951,127 @@ mod tests {Ok(())}+ #[test]+ fn case_without_expr_with_custom_column() -> Result<()> {+ let batch = case_test_batch()?;+ let schema = batch.schema();++ let when1 = binary(+ Arc::new(CustomColumn::new("a", 0)),+ Operator::Eq,+ lit("foo"),+ &schema,+ )?;+ let when2 = binary(+ Arc::new(CustomColumn::new("a", 0)),+ Operator::Eq,+ lit("bar"),+ &schema,+ )?;+ let expr = generate_case_when_with_type_coercion(+ None,+ vec![(when1, lit(123i32)), (when2, lit(456i32))],+ None,+ schema.as_ref(),+ )?;++ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ let result = as_int32_array(&result)?;+ let expected = Int32Array::from(vec![Some(123), None, None, Some(456)]);++ assert_eq!(&expected, result);+ Ok(())+ }++ #[test]+ fn case_with_expr_with_custom_column() -> Result<()> {+ let batch = case_test_batch_with_extra_columns()?;+ let schema = batch.schema();++ let expr = CaseExpr::try_new(+ Some(Arc::new(CustomColumn::new("a", 0))),+ vec![(lit("foo"), col("b", &schema)?)],+ Some(col("c", &schema)?),+ )?;++ match &expr.eval_method {+ EvalMethod::WithExpression(projected) => {+ assert!(!projected.supports_projection);+ }+ method => panic!("expected WithExpression, got {method:?}"),+ }++ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ let result = as_int32_array(&result)?;+ let expected = Int32Array::from(vec![Some(1), Some(20), Some(30), Some(40)]);++ assert_eq!(&expected, result);+ Ok(())+ }++ #[test]+ fn case_expr_or_expr_with_custom_column() -> Result<()> {+ let batch = case_test_batch_with_extra_columns()?;+ let schema = batch.schema();+ let when = binary(+ Arc::new(CustomColumn::new("a", 0)),+ Operator::Eq,+ lit("foo"),+ &schema,+ )?;++ let expr = CaseExpr::try_new(+ None,+ vec![(when, col("b", &schema)?)],+ Some(col("c", &schema)?),+ )?;++ match &expr.eval_method {+ EvalMethod::ExpressionOrExpression(projected) => {+ assert!(!projected.supports_projection);+ }+ method => panic!("expected ExpressionOrExpression, got {method:?}"),+ }++ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ let result = as_int32_array(&result)?;+ let expected = Int32Array::from(vec![Some(1), Some(20), Some(30), Some(40)]);++ assert_eq!(&expected, result);+ Ok(())+ }++ #[test]+ fn case_expr_or_expr_keeps_nullary_scalar_functions_projectable() -> Result<()> {+ let batch = case_test_batch1()?;+ let schema = batch.schema();+ let random: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::try_new(+ datafusion_functions::math::random(),+ vec![],+ &schema,+ Arc::new(ConfigOptions::default()),+ )?);+ let when = binary(random, Operator::Gt, lit(0.5f64), &schema)?;++ let expr = CaseExpr::try_new(+ None,+ vec![(when, col("b", &schema)?)],+ Some(col("c", &schema)?),+ )?;++ match &expr.eval_method {+ EvalMethod::ExpressionOrExpression(projected) => {+ assert!(projected.supports_projection);+ assert_eq!(projected.projection, vec![1, 2]);+ }+ method => panic!("expected ExpressionOrExpression, got {method:?}"),+ }++ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ assert_eq!(result.len(), batch.num_rows());+ Ok(())+ }+#[test]fn case_with_expr_when_null() -> Result<()> {let batch = case_test_batch()?;@@ -1961,6 +2139,22 @@ mod tests {Ok(())}+ fn case_test_batch_with_extra_columns() -> Result<RecordBatch> {+ let schema = Schema::new(vec![+ Field::new("a", DataType::Utf8, true),+ Field::new("b", DataType::Int32, true),+ Field::new("c", DataType::Int32, true),+ ]);+ let a = StringArray::from(vec![Some("foo"), Some("baz"), None, Some("bar")]);+ let b = Int32Array::from(vec![Some(1), Some(2), Some(3), Some(4)]);+ let c = Int32Array::from(vec![Some(10), Some(20), Some(30), Some(40)]);+ RecordBatch::try_new(+ Arc::new(schema),+ vec![Arc::new(a), Arc::new(b), Arc::new(c)],+ )+ .map_err(Into::into)+ }+fn case_test_batch1() -> Result<RecordBatch> {let schema = Schema::new(vec![Field::new("a", DataType::Int32, true),