Apache DataFusion SQL Query Engine
Support '>', '<', '>=', '<=', '<>' in any operator (#20830)
Burak Şenbe273d69e118
diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rsindex 79d2bd6ad8..fefbf134bd 100644--- a/datafusion/sql/src/expr/mod.rs+++ b/datafusion/sql/src/expr/mod.rs@@ -36,11 +36,11 @@ use datafusion_expr::expr::SetQuantifier;use datafusion_expr::expr::{InList, WildcardOptions};use datafusion_expr::{Between, BinaryExpr, Cast, Expr, ExprSchemable, GetFieldAccess, Like, Literal,- Operator, TryCast, lit,+ Operator, TryCast, lit, when,};use crate::planner::{ContextProvider, PlannerContext, SqlToRel};-use datafusion_functions_nested::expr_fn::array_has;+use datafusion_functions_nested::expr_fn::{array_has, array_max, array_min};mod binary_op;mod function;@@ -608,17 +608,9 @@ impl<S: ContextProvider> SqlToRel<'_, S> {planner_context,),_ => {- if compare_op != BinaryOperator::Eq {- plan_err!(- "Unsupported AnyOp: '{compare_op}', only '=' is supported"- )- } else {- let left_expr =- self.sql_to_expr(*left, schema, planner_context)?;- let right_expr =- self.sql_to_expr(*right, schema, planner_context)?;- Ok(array_has(right_expr, left_expr))- }+ let left_expr = self.sql_to_expr(*left, schema, planner_context)?;+ let right_expr = self.sql_to_expr(*right, schema, planner_context)?;+ plan_any_op(left_expr, right_expr, &compare_op)}},SQLExpr::AllOp {@@ -1242,6 +1234,60 @@ impl<S: ContextProvider> SqlToRel<'_, S> {}}+/// Builds a CASE expression that handles NULL semantics for `x <op> ANY(arr)`:+///+/// ```text+/// CASE+/// WHEN <min_or_max>(arr) IS NOT NULL THEN <comparison>+/// WHEN arr IS NOT NULL THEN FALSE -- empty or all-null array+/// ELSE NULL -- NULL array+/// END+/// ```+fn any_op_with_null_handling(bound: Expr, comparison: Expr, arr: Expr) -> Result<Expr> {+ when(bound.is_not_null(), comparison)+ .when(arr.is_not_null(), lit(false))+ .otherwise(lit(ScalarValue::Boolean(None)))+}++/// Plans a `<left> <op> ANY(<right>)` expression for non-subquery operands.+fn plan_any_op(+ left_expr: Expr,+ right_expr: Expr,+ compare_op: &BinaryOperator,+) -> Result<Expr> {+ match compare_op {+ BinaryOperator::Eq => Ok(array_has(right_expr, left_expr)),+ BinaryOperator::NotEq => {+ let min = array_min(right_expr.clone());+ let max = array_max(right_expr.clone());+ // NOT EQ is true when either bound differs from left+ let comparison = min+ .not_eq(left_expr.clone())+ .or(max.clone().not_eq(left_expr));+ any_op_with_null_handling(max, comparison, right_expr)+ }+ BinaryOperator::Gt => {+ let min = array_min(right_expr.clone());+ any_op_with_null_handling(min.clone(), min.lt(left_expr), right_expr)+ }+ BinaryOperator::Lt => {+ let max = array_max(right_expr.clone());+ any_op_with_null_handling(max.clone(), max.gt(left_expr), right_expr)+ }+ BinaryOperator::GtEq => {+ let min = array_min(right_expr.clone());+ any_op_with_null_handling(min.clone(), min.lt_eq(left_expr), right_expr)+ }+ BinaryOperator::LtEq => {+ let max = array_max(right_expr.clone());+ any_op_with_null_handling(max.clone(), max.gt_eq(left_expr), right_expr)+ }+ _ => plan_err!(+ "Unsupported AnyOp: '{compare_op}', only '=', '<>', '>', '<', '>=', '<=' are supported"+ ),+ }+}+#[cfg(test)]mod tests {use std::collections::HashMap;diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.sltindex 7e6050d8e6..91efdd63a7 100644--- a/datafusion/sqllogictest/test_files/array.slt+++ b/datafusion/sqllogictest/test_files/array.slt@@ -7101,8 +7101,244 @@ select count(*) from arrays where 'X'=any(column3);----0-query error DataFusion error: Error during planning: Unsupported AnyOp: '>', only '=' is supported-select count(*) from arrays where 'X'>any(column3);+# any operator with comparison operators+# Use inline arrays so the test data is visible and the needle (5)+# falls within the range of some arrays but not others.+statement ok+CREATE TABLE any_op_test AS VALUES+ (1, make_array(1, 2, 3)),+ (2, make_array(4, 5, 6)),+ (3, make_array(7, 8, 9)),+ (4, make_array(3, 5, 7));++# 5 > ANY(arr): true when array_min < 5+# row1: min=1 < 5 ✓, row2: min=4 < 5 ✓, row3: min=7 < 5 ✗, row4: min=3 < 5 ✓+query I?+select column1, column2 from any_op_test where 5 > any(column2) order by column1;+----+1 [1, 2, 3]+2 [4, 5, 6]+4 [3, 5, 7]++# 5 >= ANY(arr): true when array_min <= 5+# row1: min=1 <= 5 ✓, row2: min=4 <= 5 ✓, row3: min=7 <= 5 ✗, row4: min=3 <= 5 ✓+query I?+select column1, column2 from any_op_test where 5 >= any(column2) order by column1;+----+1 [1, 2, 3]+2 [4, 5, 6]+4 [3, 5, 7]++# 5 < ANY(arr): true when array_max > 5+# row1: max=3 > 5 ✗, row2: max=6 > 5 ✓, row3: max=9 > 5 ✓, row4: max=7 > 5 ✓+query I?+select column1, column2 from any_op_test where 5 < any(column2) order by column1;+----+2 [4, 5, 6]+3 [7, 8, 9]+4 [3, 5, 7]++# 5 <= ANY(arr): true when array_max >= 5+# row1: max=3 >= 5 ✗, row2: max=6 >= 5 ✓, row3: max=9 >= 5 ✓, row4: max=7 >= 5 ✓+query I?+select column1, column2 from any_op_test where 5 <= any(column2) order by column1;+----+2 [4, 5, 6]+3 [7, 8, 9]+4 [3, 5, 7]++# 5 <> ANY(arr): true when array_min != 5 OR array_max != 5+# row1: [1,2,3] min=1!=5 ✓, row2: [4,5,6] min=4!=5 ✓, row3: [7,8,9] min=7!=5 ✓, row4: [3,5,7] min=3!=5 ✓+query I?+select column1, column2 from any_op_test where 5 <> any(column2) order by column1;+----+1 [1, 2, 3]+2 [4, 5, 6]+3 [7, 8, 9]+4 [3, 5, 7]++# For a single-element array where the element equals the needle, <> should return false+query B+select 5 <> any(make_array(5));+----+false++# For a uniform array [5,5,5], <> should also return false+query B+select 5 <> any(make_array(5, 5, 5));+----+false++# Empty array: all operators should return false (no elements satisfy the condition)+query B+select 5 = any(make_array());+----+false++query B+select 5 <> any(make_array());+----+false++query B+select 5 > any(make_array());+----+false++query B+select 5 < any(make_array());+----+false++query B+select 5 >= any(make_array());+----+false++query B+select 5 <= any(make_array());+----+false++# Mixed NULL + non-NULL array where no non-NULL element satisfies the condition+# These return false (NULLs are skipped by array_min/array_max)+query B+select 5 > any(make_array(6, NULL));+----+false++query B+select 5 < any(make_array(3, NULL));+----+false++query B+select 5 >= any(make_array(6, NULL));+----+false++query B+select 5 <= any(make_array(3, NULL));+----+false++# Mixed NULL + non-NULL array where a non-NULL element satisfies the condition+query B+select 5 > any(make_array(3, NULL));+----+true++query B+select 5 < any(make_array(6, NULL));+----+true++query B+select 5 >= any(make_array(5, NULL));+----+true++query B+select 5 <= any(make_array(5, NULL));+----+true++query B+select 5 <> any(make_array(3, NULL));+----+true++query B+select 5 <> any(make_array(5, NULL));+----+false++# All-NULL array: all operators should return false+query B+select 5 > any(make_array(NULL::INT, NULL::INT));+----+false++query B+select 5 < any(make_array(NULL::INT, NULL::INT));+----+false++query B+select 5 >= any(make_array(NULL::INT, NULL::INT));+----+false++query B+select 5 <= any(make_array(NULL::INT, NULL::INT));+----+false++query B+select 5 <> any(make_array(NULL::INT, NULL::INT));+----+false++# NULL left operand: should return NULL for non-empty arrays+query B+select NULL > any(make_array(1, 2, 3));+----+NULL++query B+select NULL < any(make_array(1, 2, 3));+----+NULL++query B+select NULL >= any(make_array(1, 2, 3));+----+NULL++query B+select NULL <= any(make_array(1, 2, 3));+----+NULL++query B+select NULL <> any(make_array(1, 2, 3));+----+NULL++# NULL left operand with empty array: should return false+query B+select NULL > any(make_array());+----+false++# NULL array: should return NULL+query B+select 5 > any(NULL::INT[]);+----+NULL++query B+select 5 < any(NULL::INT[]);+----+NULL++query B+select 5 >= any(NULL::INT[]);+----+NULL++query B+select 5 <= any(NULL::INT[]);+----+NULL++query B+select 5 <> any(NULL::INT[]);+----+NULL++statement ok+DROP TABLE any_op_test;## array_distinct