Skip to content

Apache DataFusion SQL Query Engine

Fix CASE evaluation for custom column expressions (#24484)

Hasnaat hussain69a4c72264a3
diff --git a/datafusion/physical-expr/src/expressions/case.rs b/datafusion/physical-expr/src/expressions/case.rs
index e95e9e570c..dd98029c73 100644
--- a/datafusion/physical-expr/src/expressions/case.rs
+++ b/datafusion/physical-expr/src/expressions/case.rs
@@ -18,10 +18,10 @@
mod literal_lookup_table;
use super::{Column, Literal};
-use crate::PhysicalExpr;
use crate::expressions::{
CastExpr, LambdaVariable, NegativeExpr, NotExpr, lit, try_cast,
};
+use crate::{PhysicalExpr, ScalarFunctionExpr};
use arrow::array::*;
use arrow::compute::kernels::zip::zip;
use arrow::compute::{
@@ -133,6 +133,7 @@ impl CaseBody {
// Determine the set of columns that are used in all the expressions of the case body.
// Use an ordered set so lambda variables continue to be positioned after columns
let mut used_column_indices = BTreeSet::<usize>::new();
+ let mut supports_projection = true;
let mut collect_column_indices = |expr: &Arc<dyn PhysicalExpr>| {
expr.apply(|expr| {
if let Some(column) = expr.downcast_ref::<Column>() {
@@ -141,6 +142,12 @@ impl CaseBody {
expr.downcast_ref::<LambdaVariable>()
{
used_column_indices.insert(lambda_variable.index());
+ } else if expr.downcast_ref::<Literal>().is_none()
+ && expr.downcast_ref::<ScalarFunctionExpr>().is_none()
+ && expr.children().is_empty()
+ {
+ // Unknown leaves may read input columns without exposing a Column child.
+ supports_projection = false;
}
Ok(TreeNodeRecursion::Continue)
})
@@ -216,6 +223,7 @@ impl CaseBody {
Ok(ProjectedCaseBody {
projection,
body: projected_body,
+ supports_projection,
})
}
}
@@ -247,10 +255,32 @@ impl CaseBody {
///
/// The projection vector and the rewritten expression (which only differs from the original in
/// column reference indices) are held in a `ProjectedCaseBody`.
+///
+/// When `supports_projection` is false, evaluation uses the original body and full batch because
+/// the expression may depend on input columns that are not visible in its children.
#[derive(Debug, Hash, PartialEq, Eq)]
struct ProjectedCaseBody {
projection: Vec<usize>,
body: CaseBody,
+ /// Whether `body` is safe to evaluate against a projected batch.
+ ///
+ /// When false, `projection` and the rewritten `body` must not be used because an
+ /// expression may read input columns without exposing them through its children.
+ supports_projection: bool,
+}
+
+impl ProjectedCaseBody {
+ /// Returns a projection when evaluating the derived body can avoid unused input columns.
+ fn projection_for(&self, batch: &RecordBatch) -> Option<Vec<usize>> {
+ let projection = self
+ .projection
+ .iter()
+ .copied()
+ .filter(|index| *index < batch.num_columns())
+ .collect::<Vec<_>>();
+ (self.supports_projection && projection.len() < batch.num_columns())
+ .then_some(projection)
+ }
}
/// The CASE expression is similar to a series of nested if/else and there are two forms that
@@ -1049,14 +1079,7 @@ impl CaseExpr {
projected: &ProjectedCaseBody,
) -> Result<ColumnarValue> {
let return_type = self.data_type(&batch.schema())?;
- // projected.projection may include indexes of lambda variables not available on this batch
- let projection = projected
- .projection
- .iter()
- .copied()
- .filter(|index| *index < batch.num_columns())
- .collect::<Vec<_>>();
- if projection.len() < batch.num_columns() {
+ if let Some(projection) = projected.projection_for(batch) {
let projected_batch = batch.project(&projection)?;
projected
.body
@@ -1079,14 +1102,7 @@ impl CaseExpr {
projected: &ProjectedCaseBody,
) -> Result<ColumnarValue> {
let return_type = self.data_type(&batch.schema())?;
- // projected.projection may include indexes of lambda variables not available on this batch
- let projection = projected
- .projection
- .iter()
- .copied()
- .filter(|index| *index < batch.num_columns())
- .collect::<Vec<_>>();
- if projection.len() < batch.num_columns() {
+ if let Some(projection) = projected.projection_for(batch) {
let projected_batch = batch.project(&projection)?;
projected
.body
@@ -1204,23 +1220,14 @@ impl CaseExpr {
)?))
}
}
+ } else if let Some(projection) = projected.projection_for(batch) {
+ // The case expressions do not use all the columns of the input batch.
+ // Project first to reduce time spent filtering.
+ let projected_batch = batch.project(&projection)?;
+ projected.body.expr_or_expr(&projected_batch, when_value)
} else {
- // projected.projection may include indexes of lambda variables not available on this batch
- let projection = projected
- .projection
- .iter()
- .copied()
- .filter(|index| *index < batch.num_columns())
- .collect::<Vec<_>>();
- if projection.len() < batch.num_columns() {
- // The case expressions do not use all the columns of the input batch.
- // Project first to reduce time spent filtering.
- let projected_batch = batch.project(&projection)?;
- projected.body.expr_or_expr(&projected_batch, when_value)
- } else {
- // All columns are used in the case expressions, so there is no need to project.
- self.body.expr_or_expr(batch, when_value)
- }
+ // All columns are used in the case expressions, so there is no need to project.
+ self.body.expr_or_expr(batch, when_value)
}
}
@@ -1592,6 +1599,7 @@ mod tests {
use arrow::datatypes::DataType::Float64;
use arrow::datatypes::Field;
use datafusion_common::cast::{as_float64_array, as_int32_array};
+ use datafusion_common::config::ConfigOptions;
use datafusion_common::plan_err;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_expr::type_coercion::binary::type_union_coercion;
@@ -1599,6 +1607,55 @@ mod tests {
use datafusion_physical_expr_common::physical_expr::fmt_sql;
use half::f16;
+ #[derive(Debug, Hash, PartialEq, Eq)]
+ struct CustomColumn {
+ inner: Column,
+ }
+
+ impl CustomColumn {
+ fn new(name: &str, index: usize) -> Self {
+ Self {
+ inner: Column::new(name, index),
+ }
+ }
+ }
+
+ impl std::fmt::Display for CustomColumn {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ std::fmt::Display::fmt(&self.inner, f)
+ }
+ }
+
+ impl PhysicalExpr for CustomColumn {
+ fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
+ self.inner.data_type(input_schema)
+ }
+
+ fn nullable(&self, input_schema: &Schema) -> Result<bool> {
+ self.inner.nullable(input_schema)
+ }
+
+ fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
+ self.inner.evaluate(batch)
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
+ vec![]
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ children: Vec<Arc<dyn PhysicalExpr>>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ assert!(children.is_empty());
+ Ok(self)
+ }
+
+ fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ self.inner.fmt_sql(f)
+ }
+ }
+
#[test]
fn case_with_expr() -> Result<()> {
let batch = case_test_batch()?;
@@ -1894,6 +1951,127 @@ mod tests {
Ok(())
}
+ #[test]
+ fn case_without_expr_with_custom_column() -> Result<()> {
+ let batch = case_test_batch()?;
+ let schema = batch.schema();
+
+ let when1 = binary(
+ Arc::new(CustomColumn::new("a", 0)),
+ Operator::Eq,
+ lit("foo"),
+ &schema,
+ )?;
+ let when2 = binary(
+ Arc::new(CustomColumn::new("a", 0)),
+ Operator::Eq,
+ lit("bar"),
+ &schema,
+ )?;
+ let expr = generate_case_when_with_type_coercion(
+ None,
+ vec![(when1, lit(123i32)), (when2, lit(456i32))],
+ None,
+ schema.as_ref(),
+ )?;
+
+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
+ let result = as_int32_array(&result)?;
+ let expected = Int32Array::from(vec![Some(123), None, None, Some(456)]);
+
+ assert_eq!(&expected, result);
+ Ok(())
+ }
+
+ #[test]
+ fn case_with_expr_with_custom_column() -> Result<()> {
+ let batch = case_test_batch_with_extra_columns()?;
+ let schema = batch.schema();
+
+ let expr = CaseExpr::try_new(
+ Some(Arc::new(CustomColumn::new("a", 0))),
+ vec![(lit("foo"), col("b", &schema)?)],
+ Some(col("c", &schema)?),
+ )?;
+
+ match &expr.eval_method {
+ EvalMethod::WithExpression(projected) => {
+ assert!(!projected.supports_projection);
+ }
+ method => panic!("expected WithExpression, got {method:?}"),
+ }
+
+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
+ let result = as_int32_array(&result)?;
+ let expected = Int32Array::from(vec![Some(1), Some(20), Some(30), Some(40)]);
+
+ assert_eq!(&expected, result);
+ Ok(())
+ }
+
+ #[test]
+ fn case_expr_or_expr_with_custom_column() -> Result<()> {
+ let batch = case_test_batch_with_extra_columns()?;
+ let schema = batch.schema();
+ let when = binary(
+ Arc::new(CustomColumn::new("a", 0)),
+ Operator::Eq,
+ lit("foo"),
+ &schema,
+ )?;
+
+ let expr = CaseExpr::try_new(
+ None,
+ vec![(when, col("b", &schema)?)],
+ Some(col("c", &schema)?),
+ )?;
+
+ match &expr.eval_method {
+ EvalMethod::ExpressionOrExpression(projected) => {
+ assert!(!projected.supports_projection);
+ }
+ method => panic!("expected ExpressionOrExpression, got {method:?}"),
+ }
+
+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
+ let result = as_int32_array(&result)?;
+ let expected = Int32Array::from(vec![Some(1), Some(20), Some(30), Some(40)]);
+
+ assert_eq!(&expected, result);
+ Ok(())
+ }
+
+ #[test]
+ fn case_expr_or_expr_keeps_nullary_scalar_functions_projectable() -> Result<()> {
+ let batch = case_test_batch1()?;
+ let schema = batch.schema();
+ let random: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::try_new(
+ datafusion_functions::math::random(),
+ vec![],
+ &schema,
+ Arc::new(ConfigOptions::default()),
+ )?);
+ let when = binary(random, Operator::Gt, lit(0.5f64), &schema)?;
+
+ let expr = CaseExpr::try_new(
+ None,
+ vec![(when, col("b", &schema)?)],
+ Some(col("c", &schema)?),
+ )?;
+
+ match &expr.eval_method {
+ EvalMethod::ExpressionOrExpression(projected) => {
+ assert!(projected.supports_projection);
+ assert_eq!(projected.projection, vec![1, 2]);
+ }
+ method => panic!("expected ExpressionOrExpression, got {method:?}"),
+ }
+
+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
+ assert_eq!(result.len(), batch.num_rows());
+ Ok(())
+ }
+
#[test]
fn case_with_expr_when_null() -> Result<()> {
let batch = case_test_batch()?;
@@ -1961,6 +2139,22 @@ mod tests {
Ok(())
}
+ fn case_test_batch_with_extra_columns() -> Result<RecordBatch> {
+ let schema = Schema::new(vec![
+ Field::new("a", DataType::Utf8, true),
+ Field::new("b", DataType::Int32, true),
+ Field::new("c", DataType::Int32, true),
+ ]);
+ let a = StringArray::from(vec![Some("foo"), Some("baz"), None, Some("bar")]);
+ let b = Int32Array::from(vec![Some(1), Some(2), Some(3), Some(4)]);
+ let c = Int32Array::from(vec![Some(10), Some(20), Some(30), Some(40)]);
+ RecordBatch::try_new(
+ Arc::new(schema),
+ vec![Arc::new(a), Arc::new(b), Arc::new(c)],
+ )
+ .map_err(Into::into)
+ }
+
fn case_test_batch1() -> Result<RecordBatch> {
let schema = Schema::new(vec![
Field::new("a", DataType::Int32, true),