Apache DataFusion SQL Query Engine
Enable inlist support for preimage (#20051)
Kosta Tarasovdf328592f1a1
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rsindex ce563378a2..c6644e0086 100644--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs@@ -2044,6 +2044,53 @@ impl TreeNodeRewriter for Simplifier<'_> {Transformed::no(Expr::BinaryExpr(BinaryExpr { left, op, right }))}}+ // For case:+ // date_part('YEAR', expr) IN (literal1, literal2, ...)+ Expr::InList(InList {+ expr,+ list,+ negated,+ }) => {+ if list.len() > THRESHOLD_INLINE_INLIST || list.iter().any(is_null) {+ return Ok(Transformed::no(Expr::InList(InList {+ expr,+ list,+ negated,+ })));+ }++ let (op, combiner): (Operator, fn(Expr, Expr) -> Expr) =+ if negated { (NotEq, and) } else { (Eq, or) };++ let mut rewritten: Option<Expr> = None;+ for item in &list {+ let PreimageResult::Range { interval, expr } =+ get_preimage(expr.as_ref(), item, info)?+ else {+ return Ok(Transformed::no(Expr::InList(InList {+ expr,+ list,+ negated,+ })));+ };++ let range_expr = rewrite_with_preimage(*interval, op, expr)?.data;+ rewritten = Some(match rewritten {+ None => range_expr,+ Some(acc) => combiner(acc, range_expr),+ });+ }++ if let Some(rewritten) = rewritten {+ Transformed::yes(rewritten)+ } else {+ Transformed::no(Expr::InList(InList {+ expr,+ list,+ negated,+ }))+ }+ }// no additional rewrites possibleexpr => Transformed::no(expr),diff --git a/datafusion/optimizer/src/simplify_expressions/udf_preimage.rs b/datafusion/optimizer/src/simplify_expressions/udf_preimage.rsindex 9d56ffaa63..da2716d13c 100644--- a/datafusion/optimizer/src/simplify_expressions/udf_preimage.rs+++ b/datafusion/optimizer/src/simplify_expressions/udf_preimage.rs@@ -75,7 +75,7 @@ mod test {use datafusion_common::{DFSchema, DFSchemaRef, Result, ScalarValue};use datafusion_expr::{ColumnarValue, Expr, Operator, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,- Signature, Volatility, and, binary_expr, col, lit, preimage::PreimageResult,+ Signature, Volatility, and, binary_expr, col, lit, or, preimage::PreimageResult,simplify::SimplifyContext,};@@ -164,6 +164,15 @@ mod test {)?),})}+ Expr::Literal(ScalarValue::Int32(Some(600)), _) => {+ Ok(PreimageResult::Range {+ expr,+ interval: Box::new(Interval::try_new(+ ScalarValue::Int32(Some(300)),+ ScalarValue::Int32(Some(400)),+ )?),+ })+ }_ => Ok(PreimageResult::None),}}@@ -311,6 +320,38 @@ mod test {assert_eq!(optimize_test(expr, &schema), expected);}+ #[test]+ fn test_preimage_in_list_rewrite() {+ let schema = test_schema();+ let expr = preimage_udf_expr().in_list(vec![lit(500), lit(600)], false);+ let expected = or(+ and(col("x").gt_eq(lit(100)), col("x").lt(lit(200))),+ and(col("x").gt_eq(lit(300)), col("x").lt(lit(400))),+ );++ assert_eq!(optimize_test(expr, &schema), expected);+ }++ #[test]+ fn test_preimage_not_in_list_rewrite() {+ let schema = test_schema();+ let expr = preimage_udf_expr().in_list(vec![lit(500), lit(600)], true);+ let expected = and(+ or(col("x").lt(lit(100)), col("x").gt_eq(lit(200))),+ or(col("x").lt(lit(300)), col("x").gt_eq(lit(400))),+ );++ assert_eq!(optimize_test(expr, &schema), expected);+ }++ #[test]+ fn test_preimage_in_list_long_list_no_rewrite() {+ let schema = test_schema();+ let expr = preimage_udf_expr().in_list((1..100).map(lit).collect(), false);++ assert_eq!(optimize_test(expr.clone(), &schema), expr);+ }+#[test]fn test_preimage_non_literal_rhs_no_rewrite() {// Non-literal RHS should not be rewritten.diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.sltindex bffcf76bbf..79d6d8ac05 100644--- a/datafusion/sqllogictest/test_files/datetime/date_part.slt+++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt@@ -1247,6 +1247,19 @@ NULL1990-01-012030-01-01+# IN list optimization+query D+select c from t1 where extract(year from c) in (1990, 2024);+----+1990-01-01+2024-01-01++# NOT IN list optimization (NULL does not satisfy NOT IN)+query D+select c from t1 where extract(year from c) not in (1990, 2024);+----+2030-01-01+# Check that date_part is not in the explain statementsquery TT@@ -1329,6 +1342,16 @@ physical_plan01)FilterExec: c@0 < 2024-01-01 OR c@0 >= 2025-01-01 OR c@0 IS NULL02)--DataSourceExec: partitions=1, partition_sizes=[1]+query TT+explain select c from t1 where extract (year from c) in (1990, 2024)+----+logical_plan+01)Filter: t1.c >= Date32("1990-01-01") AND t1.c < Date32("1991-01-01") OR t1.c >= Date32("2024-01-01") AND t1.c < Date32("2025-01-01")+02)--TableScan: t1 projection=[c]+physical_plan+01)FilterExec: c@0 >= 1990-01-01 AND c@0 < 1991-01-01 OR c@0 >= 2024-01-01 AND c@0 < 2025-01-01+02)--DataSourceExec: partitions=1, partition_sizes=[1]+# Simple optimizations, column on RHSquery D@@ -1730,4 +1753,4 @@ logical_plan02)--TableScan: t1 projection=[c]physical_plan01)FilterExec: c@0 >= 2024-01-01 AND c@0 < 2025-01-01-02)--DataSourceExec: partitions=1, partition_sizes=[1]\ No newline at end of file+02)--DataSourceExec: partitions=1, partition_sizes=[1]diff --git a/datafusion/sqllogictest/test_files/floor_preimage.slt b/datafusion/sqllogictest/test_files/floor_preimage.sltindex 86d15ad1a6..93302b3d7a 100644--- a/datafusion/sqllogictest/test_files/floor_preimage.slt+++ b/datafusion/sqllogictest/test_files/floor_preimage.slt@@ -104,6 +104,21 @@ query I rowsortSELECT id FROM test_data WHERE floor(float_val) = arrow_cast(5.5, 'Float64');----+# IN list: floor(x) IN (5, 7) matches [5.0, 6.0) and [7.0, 8.0)+query I rowsort+SELECT id FROM test_data WHERE floor(float_val) IN (arrow_cast(5, 'Float64'), arrow_cast(7, 'Float64'));+----+1+2+5++# NOT IN list: floor(x) NOT IN (5, 7) excludes matching ranges and NULLs+query I rowsort+SELECT id FROM test_data WHERE floor(float_val) NOT IN (arrow_cast(5, 'Float64'), arrow_cast(7, 'Float64'));+----+3+4+############ EXPLAIN Tests - Plan Optimization##########@@ -177,6 +192,14 @@ logical_plan01)Filter: floor(test_data.float_val) = Float64(9007199254740992)02)--TableScan: test_data projection=[id, float_val, int_val, decimal_val]+# 9. IN list: each list item is rewritten with preimage and OR-ed together+query TT+EXPLAIN SELECT * FROM test_data WHERE floor(float_val) IN (arrow_cast(5, 'Float64'), arrow_cast(7, 'Float64'));+----+logical_plan+01)Filter: test_data.float_val >= Float64(5) AND test_data.float_val < Float64(6) OR test_data.float_val >= Float64(7) AND test_data.float_val < Float64(8)+02)--TableScan: test_data projection=[id, float_val, int_val, decimal_val]+# Data correctness: floor(col) = 2^53 returns no rows (no value in test_data has floor exactly 2^53)query I rowsortSELECT id FROM test_data WHERE floor(float_val) = 9007199254740992;