Apache DataFusion SQL Query Engine
fix: avoid unsafe set rewrites for compound IN lists (#24952)
Bruno Volpato2648cdae8961
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rsindex 5436bd0921..bb4a04bb7b 100644--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs@@ -1894,6 +1894,9 @@ impl TreeNodeRewriter for Simplifier<'_> {right.as_ref(),false,false,+ ) && inlists_have_set_comparable_literals(+ left.as_ref(),+ right.as_ref(),) =>{match (*left, *right) {@@ -1941,6 +1944,9 @@ impl TreeNodeRewriter for Simplifier<'_> {right.as_ref(),false,true,+ ) && inlists_have_set_comparable_literals(+ left.as_ref(),+ right.as_ref(),) =>{match (*left, *right) {@@ -1968,6 +1974,9 @@ impl TreeNodeRewriter for Simplifier<'_> {right.as_ref(),true,false,+ ) && inlists_have_set_comparable_literals(+ left.as_ref(),+ right.as_ref(),) =>{match (*left, *right) {@@ -1995,6 +2004,9 @@ impl TreeNodeRewriter for Simplifier<'_> {right.as_ref(),true,true,+ ) && inlists_have_set_comparable_literals(+ left.as_ref(),+ right.as_ref(),) =>{match (*left, *right) {@@ -2285,12 +2297,30 @@ fn are_inlist_and_eq_and_match_neg() -> bool {match (left, right) {(Expr::InList(l), Expr::InList(r)) => {- l.expr == r.expr && l.negated == is_left_neg && r.negated == is_right_neg+ l.expr == r.expr+ && !l.expr.is_volatile()+ && !l.list.iter().chain(&r.list).any(Expr::is_volatile)+ && l.negated == is_left_neg+ && r.negated == is_right_neg}_ => false,}}+/// Structural equality can determine set membership only for non-null literals+/// whose equality agrees with runtime comparisons. Different runtime expressions+/// can evaluate to the same value, and floating-point equality can differ from+/// literal equality (for example, positive and negative zero).+fn inlists_have_set_comparable_literals(left: &Expr, right: &Expr) -> bool {+ match (left, right) {+ (Expr::InList(l), Expr::InList(r)) => l.list.iter().chain(&r.list).all(|item| {+ item.as_literal()+ .is_some_and(|value| !value.is_null() && !value.data_type().is_floating())+ }),+ _ => false,+ }+}+// TODO: We might not need this after defer pattern for Box is stabilized. https://github.com/rust-lang/rust/issues/87121fn are_inlist_and_eq(left: &Expr, right: &Expr) -> bool {let left = as_inlist(left);@@ -2299,6 +2329,7 @@ fn are_inlist_and_eq(left: &Expr, right: &Expr) -> bool {matches!(lhs.expr.as_ref(), Expr::Column(_))&& matches!(rhs.expr.as_ref(), Expr::Column(_))&& lhs.expr == rhs.expr+ && !lhs.list.iter().chain(&rhs.list).any(Expr::is_volatile)&& !lhs.negated&& !rhs.negated} else {@@ -2476,7 +2507,7 @@ mod tests {use super::*;use crate::test::test_table_scan_with_name;use arrow::{- array::{Int32Array, StructArray},+ array::{BooleanArray, Float64Array, Int32Array, StructArray},datatypes::{FieldRef, Fields},};use datafusion_common::{DFSchemaRef, ToDFSchema, assert_contains};@@ -4890,6 +4921,192 @@ mod tests {assert_eq!(simplify(expr.clone()), expr);}+ fn assert_inlist_simplification_result(+ expr: Expr,+ batch: &RecordBatch,+ expected: Vec<bool>,+ ) -> Result<()> {+ let schema = batch.schema().to_dfschema_ref()?;+ let simplifier = ExprSimplifier::new(+ SimplifyContext::builder()+ .with_schema(Arc::clone(&schema))+ .build(),+ );+ let original = simplifier.coerce(expr, &schema)?;+ let simplified = simplifier.simplify(original.clone())?;+ let props = ExecutionProps::new();+ let evaluate = |expr: &Expr| {+ create_physical_expr(+ expr,+ &schema,+ &props,+ &PhysicalPlanningContext::default(),+ )?+ .evaluate(batch)?+ .into_array(batch.num_rows())+ };+ let original_result = evaluate(&original)?;+ let actual = evaluate(&simplified)?;+ assert_eq!(original_result.as_ref(), actual.as_ref());+ assert_eq!(actual.as_boolean(), &BooleanArray::from(expected));+ Ok(())+ }++ #[test]+ fn simplify_inlist_runtime_set_operations() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("x", DataType::Int32, false),+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, false),+ ]));+ let batch = RecordBatch::try_new(+ schema,+ vec![+ Arc::new(Int32Array::from(vec![1, 2, 3])),+ Arc::new(Int32Array::from(vec![1, 1, 3])),+ Arc::new(Int32Array::from(vec![1, 2, 4])),+ ],+ )?;+ let left = |negated| {+ in_list(col("x"), vec![col("a"), lit(2), lit(10), lit(11)], negated)+ };+ let right = |negated| {+ in_list(col("x"), vec![col("b"), lit(5), lit(12), lit(13)], negated)+ };++ // Structurally different, non-null columns can hold equal values.+ for (expr, expected) in [+ (left(false).and(right(false)), vec![true, true, false]),+ (left(false).and(right(true)), vec![false, false, true]),+ (left(true).and(right(false)), vec![false, false, false]),+ (left(true).or(right(true)), vec![false, false, true]),+ // Union remains valid for nonvolatile runtime expressions.+ (left(false).or(right(false)), vec![true, true, true]),+ (left(true).and(right(true)), vec![false, false, false]),+ ] {+ assert_inlist_simplification_result(expr, &batch, expected)?;+ }++ // Coercion must precede structural comparison of mixed integer literals.+ let expr = in_list(col("x"), vec![lit(1i32), lit(2), lit(10), lit(11)], false)+ .and(in_list(+ col("x"),+ vec![lit(1i64), lit(3i64), lit(12i64), lit(13i64)],+ false,+ ));+ assert_inlist_simplification_result(expr, &batch, vec![true, false, false])+ }++ #[test]+ fn simplify_inlist_signed_zero() -> Result<()> {+ for data_type in [DataType::Float16, DataType::Float32, DataType::Float64] {+ let values =+ arrow::compute::cast(&Float64Array::from(vec![0.0, -0.0]), &data_type)?;+ let literals = |values: &[f64]| {+ values+ .iter()+ .map(|&value| {+ ScalarValue::Float64(Some(value))+ .cast_to(&data_type)+ .map(lit)+ })+ .collect::<Result<Vec<_>>>()+ };+ let left_list = literals(&[0.0, 1.0, 2.0, 3.0])?;+ let right_list = literals(&[-0.0, 4.0, 5.0, 6.0])?;+ let schema = Arc::new(Schema::new(vec![Field::new("x", data_type, false)]));+ let batch = RecordBatch::try_new(schema, vec![values])?;+ let schema = batch.schema().to_dfschema_ref()?;+ let simplifier = ExprSimplifier::new(+ SimplifyContext::builder()+ .with_schema(Arc::clone(&schema))+ .build(),+ )+ .with_canonicalize(false);+ // Cover both inlined comparisons and lists that reach the set rewrites.+ assert!(left_list.len() > THRESHOLD_INLINE_INLIST);+ assert!(right_list.len() > THRESHOLD_INLINE_INLIST);+ for list_len in [2, left_list.len()] {+ let left =+ |negated| in_list(col("x"), left_list[..list_len].to_vec(), negated);+ let right =+ |negated| in_list(col("x"), right_list[..list_len].to_vec(), negated);+ for (expr, expected) in [+ (left(false).and(right(false)), vec![true, true]),+ (left(false).and(right(true)), vec![false, false]),+ (left(true).and(right(false)), vec![false, false]),+ (left(true).or(right(true)), vec![false, false]),+ ] {+ let simplified = simplifier.simplify(expr.clone())?;+ if list_len > THRESHOLD_INLINE_INLIST {+ // Floating-point literals must bypass structural set rewrites.+ assert_eq!(simplified, expr);+ } else {+ let actual = create_physical_expr(+ &simplified,+ &schema,+ &ExecutionProps::new(),+ &PhysicalPlanningContext::default(),+ )?+ .evaluate(&batch)?+ .into_array(batch.num_rows())?;+ assert_eq!(actual.as_boolean(), &BooleanArray::from(expected));+ }+ }+ }+ }+ Ok(())+ }++ #[test]+ fn simplify_inlist_preserves_volatile_evaluations() {+ let fun = Arc::new(ScalarUDF::new_from_impl(VolatileUdf::new()));+ let volatile = Expr::ScalarFunction(ScalarFunction::new_udf(fun, vec![]));+ let left = |negated| {+ in_list(+ volatile.clone(),+ vec![lit(1), lit(2), lit(3), lit(4)],+ negated,+ )+ };+ let right = |negated| {+ in_list(+ volatile.clone(),+ vec![lit(3), lit(4), lit(5), lit(6)],+ negated,+ )+ };+ for expr in [+ left(false).and(right(false)),+ left(false).and(right(true)),+ left(true).and(right(false)),+ left(true).or(right(true)),+ left(true).and(right(true)),+ ] {+ assert_eq!(simplify_no_canonicalize(expr.clone()), expr);+ }++ // Union deduplication must not remove a repeated volatile list item.+ for negated in [false, true] {+ let left = in_list(+ col("c1"),+ vec![volatile.clone(), lit(1), lit(2), lit(3)],+ negated,+ );+ let right = in_list(+ col("c1"),+ vec![volatile.clone(), lit(4), lit(5), lit(6)],+ negated,+ );+ let expr = if negated {+ left.and(right)+ } else {+ left.or(right)+ };+ assert_eq!(simplify_no_canonicalize(expr.clone()), expr);+ }+ }+#[test]fn simplify_inlist_set_operation_propagates_nullability_errors() {let info = SimplifyContext::builder()diff --git a/datafusion/sqllogictest/test_files/predicates.slt b/datafusion/sqllogictest/test_files/predicates.sltindex 385b7f3b92..6f3865c089 100644--- a/datafusion/sqllogictest/test_files/predicates.slt+++ b/datafusion/sqllogictest/test_files/predicates.slt@@ -1029,6 +1029,27 @@ WHERE NULL NOT IN (SELECT * FROM empty); -- all rows should be returned----1+# Different non-null list expressions can evaluate to the same value.+statement ok+CREATE TABLE in_list_values(x INT NOT NULL, a INT NOT NULL, b INT NOT NULL)+AS VALUES (1, 1, 1), (2, 1, 2), (3, 3, 4);++query I rowsort+SELECT x FROM in_list_values+WHERE x IN (a, 2, 10, 11) AND x IN (b, 5, 12, 13);+----+1+2++query I+SELECT x FROM in_list_values+WHERE x IN (a, 2, 10, 11) AND x NOT IN (b, 5, 12, 13);+----+3++statement ok+DROP TABLE in_list_values;+# Config resetstatement okRESET datafusion.catalog.information_schema;