Apache DataFusion SQL Query Engine
fix: build the IN list static filter only from literal-only items (#26083)
ranflariond1371370a5c6
diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rsindex 7a7cb25317..0b3f8067ae 100644--- a/datafusion/physical-expr/src/expressions/in_list.rs+++ b/datafusion/physical-expr/src/expressions/in_list.rs@@ -22,6 +22,7 @@ use std::hash::{Hash, Hasher};use std::sync::Arc;use crate::PhysicalExpr;+use crate::expressions::Literal;use crate::physical_expr::physical_exprs_bag_equal;use arrow::array::*;@@ -31,6 +32,7 @@ use arrow::compute::kernels::boolean::{not, or_kleene};use arrow::compute::kernels::cmp::eq as arrow_eq;use arrow::datatypes::*;+use datafusion_common::tree_node::TreeNode;use datafusion_common::{DFSchema, Result, ScalarValue, assert_or_internal_err, exec_err,};@@ -107,7 +109,7 @@ fn evaluate_list(/// Try to evaluate a list of expressions as constants.////// Returns:-/// - `Ok(Some(ArrayRef))` if all expressions are constants (can be evaluated on an empty RecordBatch)+/// - `Ok(Some(ArrayRef))` if all expressions are constants (built only from literals)/// - `Ok(None)` if the list contains non-constant expressions/// - `Err(...)` only for actual errors (not for non-constant expressions)///@@ -117,6 +119,15 @@ fn try_evaluate_constant_list(list: &[Arc<dyn PhysicalExpr>],schema: &Schema,) -> Result<Option<ArrayRef>> {+ // An item can read a column yet return a scalar on an empty batch (a CASE whose+ // WHEN matches all zero rows returns its THEN), so require literal leaves.+ for expr in list {+ if expr.exists(|e| {+ Ok(e.is_volatile_node() || (e.children().is_empty() && !e.is::<Literal>()))+ })? {+ return Ok(None);+ }+ }let batch = RecordBatch::new_empty(Arc::new(schema.clone()));match evaluate_list(list, &batch) {Ok(array) => Ok(Some(array)),@@ -539,9 +550,10 @@ pub fn in_list(#[cfg(test)]mod tests {use super::*;- use crate::expressions::{col, lit, try_cast};+ use crate::expressions::{binary, case, col, lit, try_cast};use arrow::datatypes::{IntervalDayTime, IntervalMonthDayNano, i256};use datafusion_common::plan_err;+ use datafusion_expr::Operator;use datafusion_expr::type_coercion::binary::comparison_coercion;use datafusion_physical_expr_common::physical_expr::fmt_sql;use insta::assert_snapshot;@@ -3176,6 +3188,33 @@ mod tests {Ok(())}+ #[test]+ fn test_in_list_case_item_reading_a_column() -> Result<()> {+ // n IN (2, CASE WHEN d > 5 THEN 1 ELSE d END)+ let schema = Schema::new(vec![+ Field::new("n", DataType::Int32, false),+ Field::new("d", DataType::Int32, false),+ ]);+ let d = col("d", &schema)?;+ let when = binary(Arc::clone(&d), Operator::Gt, lit(5i32), &schema)?;+ let item = case(None, vec![(when, lit(1i32))], Some(d))?;+ let expr = in_list(col("n", &schema)?, vec![lit(2i32), item], &false, &schema)?;+ let batch = RecordBatch::try_new(+ Arc::new(schema),+ vec![+ Arc::new(Int32Array::from(vec![2, 4, 1])),+ Arc::new(Int32Array::from(vec![0, 4, 3])),+ ],+ )?;++ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ assert_eq!(+ as_boolean_array(&result),+ &BooleanArray::from(vec![true, true, false])+ );+ Ok(())+ }+/// Helper: creates an InListExpr with `static_filter = None`/// to force the column-reference evaluation path.fn make_in_list_with_columns(diff --git a/datafusion/sqllogictest/test_files/in_list.slt b/datafusion/sqllogictest/test_files/in_list.sltindex 3ff47afb64..8fde8b11fe 100644--- a/datafusion/sqllogictest/test_files/in_list.slt+++ b/datafusion/sqllogictest/test_files/in_list.slt@@ -827,3 +827,34 @@ ORDER BY labelmatch true falseno_match false truenulls NULL NULL++####+## List items that read a column+####++statement ok+CREATE TABLE in_list_case_items (n INT, d INT) AS VALUES (2, 0), (10, 2), (5, 2), (7, 1);++# A CASE item with a literal THEN is still evaluated per row, not once on an empty batch.+query IB+SELECT n, n IN (2, 100, 101, CASE WHEN d > 5 THEN 1 ELSE d END * 5)+FROM in_list_case_items+ORDER BY n+----+2 true+5 false+7 false+10 true++query I+SELECT n+FROM in_list_case_items+WHERE n IN (2, 100, 101, CASE WHEN d > 5 THEN 1 ELSE d END * 5)+ORDER BY n+----+2+10++# Cleanup+statement ok+DROP TABLE in_list_case_items;