Apache DataFusion SQL Query Engine
feat: support Set Comparison Subquery (#19109)
Ruihang Xia79869a75375a
diff --git a/Cargo.lock b/Cargo.lockindex 9c26b085b7..0e9337b50e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1766,6 +1766,7 @@ dependencies = ["paste","rand 0.9.2","rand_distr",+ "recursive","regex","rstest","serde",diff --git a/datafusion/catalog-listing/src/helpers.rs b/datafusion/catalog-listing/src/helpers.rsindex ea016015ce..031b2ebfb8 100644--- a/datafusion/catalog-listing/src/helpers.rs+++ b/datafusion/catalog-listing/src/helpers.rs@@ -83,6 +83,7 @@ pub fn expr_applicable_for_cols(col_names: &[&str], expr: &Expr) -> bool {| Expr::Exists(_)| Expr::InSubquery(_)| Expr::ScalarSubquery(_)+ | Expr::SetComparison(_)| Expr::GroupingSet(_)| Expr::Case(_) => Ok(TreeNodeRecursion::Continue),diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.tomlindex bd88ed3b9c..d485c04e38 100644--- a/datafusion/core/Cargo.toml+++ b/datafusion/core/Cargo.toml@@ -177,6 +177,7 @@ insta = { workspace = true }paste = { workspace = true }rand = { workspace = true, features = ["small_rng"] }rand_distr = "0.5"+recursive = { workspace = true }regex = { workspace = true }rstest = { workspace = true }serde_json = { workspace = true }diff --git a/datafusion/core/tests/set_comparison.rs b/datafusion/core/tests/set_comparison.rsnew file mode 100644index 0000000000..464d6c937b--- /dev/null+++ b/datafusion/core/tests/set_comparison.rs@@ -0,0 +1,193 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::sync::Arc;++use arrow::array::{Int32Array, StringArray};+use arrow::datatypes::{DataType, Field, Schema};+use arrow::record_batch::RecordBatch;+use datafusion::prelude::SessionContext;+use datafusion_common::{Result, assert_batches_eq, assert_contains};++fn build_table(values: &[i32]) -> Result<RecordBatch> {+ let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int32, true)]));+ let array =+ Arc::new(Int32Array::from(values.to_vec())) as Arc<dyn arrow::array::Array>;+ RecordBatch::try_new(schema, vec![array]).map_err(Into::into)+}++#[tokio::test]+async fn set_comparison_any() -> Result<()> {+ let ctx = SessionContext::new();++ ctx.register_batch("t", build_table(&[1, 6, 10])?)?;+ // Include a NULL in the subquery input to ensure we propagate UNKNOWN correctly.+ ctx.register_batch("s", {+ let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int32, true)]));+ let array = Arc::new(Int32Array::from(vec![Some(5), None]))+ as Arc<dyn arrow::array::Array>;+ RecordBatch::try_new(schema, vec![array])?+ })?;++ let df = ctx+ .sql("select v from t where v > any(select v from s)")+ .await?;+ let results = df.collect().await?;++ assert_batches_eq!(+ &["+----+", "| v |", "+----+", "| 6 |", "| 10 |", "+----+",],+ &results+ );+ Ok(())+}++#[tokio::test]+async fn set_comparison_any_aggregate_subquery() -> Result<()> {+ let ctx = SessionContext::new();++ ctx.register_batch("t", build_table(&[1, 7])?)?;+ ctx.register_batch("s", build_table(&[1, 2, 3])?)?;++ let df = ctx+ .sql(+ "select v from t where v > any(select sum(v) from s group by v % 2) order by v",+ )+ .await?;+ let results = df.collect().await?;++ assert_batches_eq!(&["+---+", "| v |", "+---+", "| 7 |", "+---+",], &results);+ Ok(())+}++#[tokio::test]+async fn set_comparison_all_empty() -> Result<()> {+ let ctx = SessionContext::new();++ ctx.register_batch("t", build_table(&[1, 6, 10])?)?;+ ctx.register_batch(+ "e",+ RecordBatch::new_empty(Arc::new(Schema::new(vec![Field::new(+ "v",+ DataType::Int32,+ true,+ )]))),+ )?;++ let df = ctx+ .sql("select v from t where v < all(select v from e)")+ .await?;+ let results = df.collect().await?;++ assert_batches_eq!(+ &[+ "+----+", "| v |", "+----+", "| 1 |", "| 6 |", "| 10 |", "+----+",+ ],+ &results+ );+ Ok(())+}++#[tokio::test]+async fn set_comparison_type_mismatch() -> Result<()> {+ let ctx = SessionContext::new();++ ctx.register_batch("t", build_table(&[1])?)?;+ ctx.register_batch("strings", {+ let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, true)]));+ let array = Arc::new(StringArray::from(vec![Some("a"), Some("b")]))+ as Arc<dyn arrow::array::Array>;+ RecordBatch::try_new(schema, vec![array])?+ })?;++ let df = ctx+ .sql("select v from t where v > any(select s from strings)")+ .await?;+ let err = df.collect().await.unwrap_err();+ assert_contains!(+ err.to_string(),+ "expr type Int32 can't cast to Utf8 in SetComparison"+ );+ Ok(())+}++#[tokio::test]+async fn set_comparison_multiple_operators() -> Result<()> {+ let ctx = SessionContext::new();++ ctx.register_batch("t", build_table(&[1, 2, 3, 4])?)?;+ ctx.register_batch("s", build_table(&[2, 3])?)?;++ let df = ctx+ .sql("select v from t where v = any(select v from s) order by v")+ .await?;+ let results = df.collect().await?;+ assert_batches_eq!(+ &["+---+", "| v |", "+---+", "| 2 |", "| 3 |", "+---+",],+ &results+ );++ let df = ctx+ .sql("select v from t where v != all(select v from s) order by v")+ .await?;+ let results = df.collect().await?;+ assert_batches_eq!(+ &["+---+", "| v |", "+---+", "| 1 |", "| 4 |", "+---+",],+ &results+ );++ let df = ctx+ .sql("select v from t where v >= all(select v from s) order by v")+ .await?;+ let results = df.collect().await?;+ assert_batches_eq!(+ &["+---+", "| v |", "+---+", "| 3 |", "| 4 |", "+---+",],+ &results+ );++ let df = ctx+ .sql("select v from t where v <= any(select v from s) order by v")+ .await?;+ let results = df.collect().await?;+ assert_batches_eq!(+ &[+ "+---+", "| v |", "+---+", "| 1 |", "| 2 |", "| 3 |", "+---+",+ ],+ &results+ );+ Ok(())+}++#[tokio::test]+async fn set_comparison_null_semantics_all() -> Result<()> {+ let ctx = SessionContext::new();++ ctx.register_batch("t", build_table(&[5])?)?;+ ctx.register_batch("s", {+ let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int32, true)]));+ let array = Arc::new(Int32Array::from(vec![Some(1), None]))+ as Arc<dyn arrow::array::Array>;+ RecordBatch::try_new(schema, vec![array])?+ })?;++ let df = ctx+ .sql("select v from t where v != all(select v from s)")+ .await?;+ let results = df.collect().await?;+ let row_count: usize = results.iter().map(|batch| batch.num_rows()).sum();+ assert_eq!(0, row_count);+ Ok(())+}diff --git a/datafusion/core/tests/sql/unparser.rs b/datafusion/core/tests/sql/unparser.rsindex 8b56bf67a2..ab1015b2d1 100644--- a/datafusion/core/tests/sql/unparser.rs+++ b/datafusion/core/tests/sql/unparser.rs@@ -47,6 +47,7 @@ use datafusion_physical_plan::ExecutionPlanProperties;use datafusion_sql::unparser::Unparser;use datafusion_sql::unparser::dialect::DefaultDialect;use itertools::Itertools;+use recursive::{set_minimum_stack_size, set_stack_allocation_size};/// Paths to benchmark query files (supports running from repo root or different working directories).const BENCHMARK_PATHS: &[&str] = &["../../benchmarks/", "./benchmarks/"];@@ -458,5 +459,8 @@ async fn test_clickbench_unparser_roundtrip() {#[tokio::test]async fn test_tpch_unparser_roundtrip() {+ // Grow stacker segments earlier to avoid deep unparser recursion overflow in q20.+ set_minimum_stack_size(512 * 1024);+ set_stack_allocation_size(8 * 1024 * 1024);run_roundtrip_tests("TPC-H", tpch_queries(), tpch_test_context).await;}diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rsindex 8234247c92..454839fdb7 100644--- a/datafusion/expr/src/expr.rs+++ b/datafusion/expr/src/expr.rs@@ -372,6 +372,8 @@ pub enum Expr {Exists(Exists),/// IN subqueryInSubquery(InSubquery),+ /// Set comparison subquery (e.g. `= ANY`, `> ALL`)+ SetComparison(SetComparison),/// Scalar subqueryScalarSubquery(Subquery),/// Represents a reference to all available fields in a specific schema,@@ -1101,6 +1103,54 @@ impl Exists {}}+/// Whether the set comparison uses `ANY`/`SOME` or `ALL`+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Hash, Debug)]+pub enum SetQuantifier {+ /// `ANY` (or `SOME`)+ Any,+ /// `ALL`+ All,+}++impl Display for SetQuantifier {+ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {+ match self {+ SetQuantifier::Any => write!(f, "ANY"),+ SetQuantifier::All => write!(f, "ALL"),+ }+ }+}++/// Set comparison subquery (e.g. `= ANY`, `> ALL`)+#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]+pub struct SetComparison {+ /// The expression to compare+ pub expr: Box<Expr>,+ /// Subquery that will produce a single column of data to compare against+ pub subquery: Subquery,+ /// Comparison operator (e.g. `=`, `>`, `<`)+ pub op: Operator,+ /// Quantifier (`ANY`/`ALL`)+ pub quantifier: SetQuantifier,+}++impl SetComparison {+ /// Create a new set comparison expression+ pub fn new(+ expr: Box<Expr>,+ subquery: Subquery,+ op: Operator,+ quantifier: SetQuantifier,+ ) -> Self {+ Self {+ expr,+ subquery,+ op,+ quantifier,+ }+ }+}+/// InList expression#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]pub struct InList {@@ -1502,6 +1552,7 @@ impl Expr {Expr::GroupingSet(..) => "GroupingSet",Expr::InList { .. } => "InList",Expr::InSubquery(..) => "InSubquery",+ Expr::SetComparison(..) => "SetComparison",Expr::IsNotNull(..) => "IsNotNull",Expr::IsNull(..) => "IsNull",Expr::Like { .. } => "Like",@@ -2057,6 +2108,7 @@ impl Expr {| Expr::GroupingSet(..)| Expr::InList(..)| Expr::InSubquery(..)+ | Expr::SetComparison(..)| Expr::IsFalse(..)| Expr::IsNotFalse(..)| Expr::IsNotNull(..)@@ -2650,6 +2702,16 @@ impl HashNode for Expr {subquery.hash(state);negated.hash(state);}+ Expr::SetComparison(SetComparison {+ expr: _,+ subquery,+ op,+ quantifier,+ }) => {+ subquery.hash(state);+ op.hash(state);+ quantifier.hash(state);+ }Expr::ScalarSubquery(subquery) => {subquery.hash(state);}@@ -2840,6 +2902,12 @@ impl Display for SchemaDisplay<'_> {write!(f, "NOT IN")}Expr::InSubquery(InSubquery { negated: false, .. }) => write!(f, "IN"),+ Expr::SetComparison(SetComparison {+ expr,+ op,+ quantifier,+ ..+ }) => write!(f, "{} {op} {quantifier}", SchemaDisplay(expr.as_ref())),Expr::IsTrue(expr) => write!(f, "{} IS TRUE", SchemaDisplay(expr)),Expr::IsFalse(expr) => write!(f, "{} IS FALSE", SchemaDisplay(expr)),Expr::IsNotTrue(expr) => {@@ -3315,6 +3383,12 @@ impl Display for Expr {subquery,negated: false,}) => write!(f, "{expr} IN ({subquery:?})"),+ Expr::SetComparison(SetComparison {+ expr,+ subquery,+ op,+ quantifier,+ }) => write!(f, "{expr} {op} {quantifier} ({subquery:?})"),Expr::ScalarSubquery(subquery) => write!(f, "({subquery:?})"),Expr::BinaryExpr(expr) => write!(f, "{expr}"),Expr::ScalarFunction(fun) => {@@ -3798,6 +3872,7 @@ mod test {}use super::*;+ use crate::logical_plan::{EmptyRelation, LogicalPlan};#[test]fn test_display_wildcard() {@@ -3888,6 +3963,28 @@ mod test {)}+ #[test]+ fn test_display_set_comparison() {+ let subquery = Subquery {+ subquery: Arc::new(LogicalPlan::EmptyRelation(EmptyRelation {+ produce_one_row: false,+ schema: Arc::new(DFSchema::empty()),+ })),+ outer_ref_columns: vec![],+ spans: Spans::new(),+ };++ let expr = Expr::SetComparison(SetComparison::new(+ Box::new(Expr::Column(Column::from_name("a"))),+ subquery,+ Operator::Gt,+ SetQuantifier::Any,+ ));++ assert_eq!(format!("{expr}"), "a > ANY (<subquery>)");+ assert_eq!(format!("{}", expr.human_display()), "a > ANY (<subquery>)");+ }+#[test]fn test_schema_display_alias_with_relation() {assert_eq!(diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rsindex 854e907d68..f1c77eb525 100644--- a/datafusion/expr/src/expr_schema.rs+++ b/datafusion/expr/src/expr_schema.rs@@ -195,6 +195,7 @@ impl ExprSchemable for Expr {| Expr::IsNull(_)| Expr::Exists { .. }| Expr::InSubquery(_)+ | Expr::SetComparison(_)| Expr::Between { .. }| Expr::InList { .. }| Expr::IsNotNull(_)@@ -372,6 +373,7 @@ impl ExprSchemable for Expr {| Expr::IsNotFalse(_)| Expr::IsNotUnknown(_)| Expr::Exists { .. } => Ok(false),+ Expr::SetComparison(_) => Ok(true),Expr::InSubquery(InSubquery { expr, .. }) => expr.nullable(input_schema),Expr::ScalarSubquery(subquery) => {Ok(subquery.subquery.schema().field(0).is_nullable())@@ -626,6 +628,7 @@ impl ExprSchemable for Expr {| Expr::TryCast(_)| Expr::InList(_)| Expr::InSubquery(_)+ | Expr::SetComparison(_)| Expr::Wildcard { .. }| Expr::GroupingSet(_)| Expr::Placeholder(_)diff --git a/datafusion/expr/src/logical_plan/invariants.rs b/datafusion/expr/src/logical_plan/invariants.rsindex 762491a255..b39b23e30f 100644--- a/datafusion/expr/src/logical_plan/invariants.rs+++ b/datafusion/expr/src/logical_plan/invariants.rs@@ -22,7 +22,7 @@ use datafusion_common::{use crate::{Aggregate, Expr, Filter, Join, JoinType, LogicalPlan, Window,- expr::{Exists, InSubquery},+ expr::{Exists, InSubquery, SetComparison},expr_rewriter::strip_outer_reference,utils::{collect_subquery_cols, split_conjunction},};@@ -81,6 +81,7 @@ fn assert_valid_extension_nodes(plan: &LogicalPlan, check: InvariantLevel) -> Rematch expr {Expr::Exists(Exists { subquery, .. })| Expr::InSubquery(InSubquery { subquery, .. })+ | Expr::SetComparison(SetComparison { subquery, .. })| Expr::ScalarSubquery(subquery) => {assert_valid_extension_nodes(&subquery.subquery, check)?;}@@ -133,6 +134,7 @@ fn assert_subqueries_are_valid(plan: &LogicalPlan) -> Result<()> {match expr {Expr::Exists(Exists { subquery, .. })| Expr::InSubquery(InSubquery { subquery, .. })+ | Expr::SetComparison(SetComparison { subquery, .. })| Expr::ScalarSubquery(subquery) => {check_subquery_expr(plan, &subquery.subquery, expr)?;}@@ -229,6 +231,20 @@ pub fn check_subquery_expr();}}+ if let Expr::SetComparison(set_comparison) = expr+ && set_comparison.subquery.subquery.schema().fields().len() > 1+ {+ return plan_err!(+ "Set comparison subquery should only return one column, but found {}: {}",+ set_comparison.subquery.subquery.schema().fields().len(),+ set_comparison+ .subquery+ .subquery+ .schema()+ .field_names()+ .join(", ")+ );+ }match outer_plan {LogicalPlan::Projection(_)| LogicalPlan::Filter(_)@@ -237,7 +253,7 @@ pub fn check_subquery_expr(| LogicalPlan::Aggregate(_)| LogicalPlan::Join(_) => Ok(()),_ => plan_err!(- "In/Exist subquery can only be used in \+ "In/Exist/SetComparison subquery can only be used in \Projection, Filter, TableScan, Window functions, Aggregate and Join plan nodes, \but was used in [{}]",outer_plan.display()diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rsindex ac193e758c..a1285510da 100644--- a/datafusion/expr/src/logical_plan/tree_node.rs+++ b/datafusion/expr/src/logical_plan/tree_node.rs@@ -46,7 +46,7 @@ use crate::{};use datafusion_common::tree_node::TreeNodeRefContainer;-use crate::expr::{Exists, InSubquery};+use crate::expr::{Exists, InSubquery, SetComparison};use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeContainer, TreeNodeIterator, TreeNodeRecursion,TreeNodeRewriter, TreeNodeVisitor,@@ -819,6 +819,7 @@ impl LogicalPlan {expr.apply(|expr| match expr {Expr::Exists(Exists { subquery, .. })| Expr::InSubquery(InSubquery { subquery, .. })+ | Expr::SetComparison(SetComparison { subquery, .. })| Expr::ScalarSubquery(subquery) => {// use a synthetic plan so the collector sees a// LogicalPlan::Subquery (even though it is@@ -860,6 +861,22 @@ impl LogicalPlan {})),_ => internal_err!("Transformation should return Subquery"),}),+ Expr::SetComparison(SetComparison {+ expr,+ subquery,+ op,+ quantifier,+ }) => f(LogicalPlan::Subquery(subquery))?.map_data(|s| match s {+ LogicalPlan::Subquery(subquery) => {+ Ok(Expr::SetComparison(SetComparison {+ expr,+ subquery,+ op,+ quantifier,+ }))+ }+ _ => internal_err!("Transformation should return Subquery"),+ }),Expr::ScalarSubquery(subquery) => f(LogicalPlan::Subquery(subquery))?.map_data(|s| match s {LogicalPlan::Subquery(subquery) => {diff --git a/datafusion/expr/src/planner.rs b/datafusion/expr/src/planner.rsindex 76961a49a4..042c48ffff 100644--- a/datafusion/expr/src/planner.rs+++ b/datafusion/expr/src/planner.rs@@ -249,13 +249,6 @@ pub trait ExprPlanner: Debug + Send + Sync {)}- /// Plans `ANY` expression, such as `expr = ANY(array_expr)`- ///- /// Returns origin binary expression if not possible- fn plan_any(&self, expr: RawBinaryExpr) -> Result<PlannerResult<RawBinaryExpr>> {- Ok(PlannerResult::Original(expr))- }-/// Plans aggregate functions, such as `COUNT(<expr>)`////// Returns original expression arguments if not possiblediff --git a/datafusion/expr/src/tree_node.rs b/datafusion/expr/src/tree_node.rsindex 742bae5b23..226c512a97 100644--- a/datafusion/expr/src/tree_node.rs+++ b/datafusion/expr/src/tree_node.rs@@ -20,8 +20,8 @@use crate::Expr;use crate::expr::{AggregateFunction, AggregateFunctionParams, Alias, Between, BinaryExpr, Case, Cast,- GroupingSet, InList, InSubquery, Like, Placeholder, ScalarFunction, TryCast, Unnest,- WindowFunction, WindowFunctionParams,+ GroupingSet, InList, InSubquery, Like, Placeholder, ScalarFunction, SetComparison,+ TryCast, Unnest, WindowFunction, WindowFunctionParams,};use datafusion_common::Result;@@ -58,7 +58,8 @@ impl TreeNode for Expr {| Expr::Negative(expr)| Expr::Cast(Cast { expr, .. })| Expr::TryCast(TryCast { expr, .. })- | Expr::InSubquery(InSubquery { expr, .. }) => expr.apply_elements(f),+ | Expr::InSubquery(InSubquery { expr, .. })+ | Expr::SetComparison(SetComparison { expr, .. }) => expr.apply_elements(f),Expr::GroupingSet(GroupingSet::Rollup(exprs))| Expr::GroupingSet(GroupingSet::Cube(exprs)) => exprs.apply_elements(f),Expr::ScalarFunction(ScalarFunction { args, .. }) => {@@ -128,6 +129,19 @@ impl TreeNode for Expr {| Expr::ScalarSubquery(_)| Expr::ScalarVariable(_, _)| Expr::Literal(_, _) => Transformed::no(self),+ Expr::SetComparison(SetComparison {+ expr,+ subquery,+ op,+ quantifier,+ }) => expr.map_elements(f)?.update_data(|expr| {+ Expr::SetComparison(SetComparison {+ expr,+ subquery,+ op,+ quantifier,+ })+ }),Expr::Unnest(Unnest { expr, .. }) => expr.map_elements(f)?.update_data(|expr| Expr::Unnest(Unnest { expr })),diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rsindex de4ebf5fa9..7d13c9c1d2 100644--- a/datafusion/expr/src/utils.rs+++ b/datafusion/expr/src/utils.rs@@ -312,6 +312,7 @@ pub fn expr_to_columns(expr: &Expr, accum: &mut HashSet<Column>) -> Result<()> {| Expr::InList { .. }| Expr::Exists { .. }| Expr::InSubquery(_)+ | Expr::SetComparison(_)| Expr::ScalarSubquery(_)| Expr::Wildcard { .. }| Expr::Placeholder(_)diff --git a/datafusion/functions-nested/src/planner.rs b/datafusion/functions-nested/src/planner.rsindex afb18a44f4..e96fdb7d4b 100644--- a/datafusion/functions-nested/src/planner.rs+++ b/datafusion/functions-nested/src/planner.rs@@ -37,7 +37,7 @@ use std::sync::Arc;use crate::map::map_udf;use crate::{- array_has::{array_has_all, array_has_udf},+ array_has::array_has_all,expr_fn::{array_append, array_concat, array_prepend},extract::{array_element, array_slice},make_array::make_array,@@ -120,20 +120,6 @@ impl ExprPlanner for NestedFunctionPlanner {ScalarFunction::new_udf(map_udf(), vec![keys, values]),)))}-- fn plan_any(&self, expr: RawBinaryExpr) -> Result<PlannerResult<RawBinaryExpr>> {- if expr.op == BinaryOperator::Eq {- Ok(PlannerResult::Planned(Expr::ScalarFunction(- ScalarFunction::new_udf(- array_has_udf(),- // left and right are reversed here so `needle=any(haystack)` -> `array_has(haystack, needle)`- vec![expr.right, expr.left],- ),- )))- } else {- plan_err!("Unsupported AnyOp: '{}', only '=' is supported", expr.op)- }- }}#[derive(Debug)]diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rsindex 02395c76bd..5ce2c3b6af 100644--- a/datafusion/optimizer/src/analyzer/type_coercion.rs+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs@@ -36,7 +36,7 @@ use datafusion_common::{};use datafusion_expr::expr::{self, AggregateFunctionParams, Alias, Between, BinaryExpr, Case, Exists, InList,- InSubquery, Like, ScalarFunction, Sort, WindowFunction,+ InSubquery, Like, ScalarFunction, SetComparison, Sort, WindowFunction,};use datafusion_expr::expr_rewriter::coerce_plan_expr_for_schema;use datafusion_expr::expr_schema::cast_subquery;@@ -500,6 +500,45 @@ impl TreeNodeRewriter for TypeCoercionRewriter<'_> {negated,))))}+ Expr::SetComparison(SetComparison {+ expr,+ subquery,+ op,+ quantifier,+ }) => {+ let new_plan = analyze_internal(+ self.schema,+ Arc::unwrap_or_clone(subquery.subquery),+ )?+ .data;+ let expr_type = expr.get_type(self.schema)?;+ let subquery_type = new_plan.schema().field(0).data_type();+ if (expr_type.is_numeric()+ && is_utf8_or_utf8view_or_large_utf8(subquery_type))+ || (subquery_type.is_numeric()+ && is_utf8_or_utf8view_or_large_utf8(&expr_type))+ {+ return plan_err!(+ "expr type {expr_type} can't cast to {subquery_type} in SetComparison"+ );+ }+ let common_type = comparison_coercion(&expr_type, subquery_type).ok_or(+ plan_datafusion_err!(+ "expr type {expr_type} can't cast to {subquery_type} in SetComparison"+ ),+ )?;+ let new_subquery = Subquery {+ subquery: Arc::new(new_plan),+ outer_ref_columns: subquery.outer_ref_columns,+ spans: subquery.spans,+ };+ Ok(Transformed::yes(Expr::SetComparison(SetComparison::new(+ Box::new(expr.cast_to(&common_type, self.schema)?),+ cast_subquery(new_subquery, &common_type)?,+ op,+ quantifier,+ ))))+ }Expr::Not(expr) => Ok(Transformed::yes(not(get_casted_expr_for_bool_op(*expr,self.schema,diff --git a/datafusion/optimizer/src/lib.rs b/datafusion/optimizer/src/lib.rsindex a1a59cb348..f8ab453591 100644--- a/datafusion/optimizer/src/lib.rs+++ b/datafusion/optimizer/src/lib.rs@@ -66,6 +66,7 @@ pub mod propagate_empty_relation;pub mod push_down_filter;pub mod push_down_limit;pub mod replace_distinct_aggregate;+pub mod rewrite_set_comparison;pub mod scalar_subquery_to_join;pub mod simplify_expressions;pub mod single_distinct_to_groupby;diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rsindex 8740ab072a..877a84fe4d 100644--- a/datafusion/optimizer/src/optimizer.rs+++ b/datafusion/optimizer/src/optimizer.rs@@ -51,6 +51,7 @@ use crate::propagate_empty_relation::PropagateEmptyRelation;use crate::push_down_filter::PushDownFilter;use crate::push_down_limit::PushDownLimit;use crate::replace_distinct_aggregate::ReplaceDistinctWithAggregate;+use crate::rewrite_set_comparison::RewriteSetComparison;use crate::scalar_subquery_to_join::ScalarSubqueryToJoin;use crate::simplify_expressions::SimplifyExpressions;use crate::single_distinct_to_groupby::SingleDistinctToGroupBy;@@ -235,6 +236,7 @@ impl Optimizer {/// Create a new optimizer using the recommended list of rulespub fn new() -> Self {let rules: Vec<Arc<dyn OptimizerRule + Sync + Send>> = vec![+ Arc::new(RewriteSetComparison::new()),Arc::new(OptimizeUnions::new()),Arc::new(SimplifyExpressions::new()),Arc::new(ReplaceDistinctWithAggregate::new()),diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rsindex 755ffdbafc..1ff8bdfeff 100644--- a/datafusion/optimizer/src/push_down_filter.rs+++ b/datafusion/optimizer/src/push_down_filter.rs@@ -263,6 +263,7 @@ fn can_evaluate_as_join_condition(predicate: &Expr) -> Result<bool> {| Expr::ScalarVariable(_, _) => Ok(TreeNodeRecursion::Jump),Expr::Exists { .. }| Expr::InSubquery(_)+ | Expr::SetComparison(_)| Expr::ScalarSubquery(_)| Expr::OuterReferenceColumn(_, _)| Expr::Unnest(_) => {diff --git a/datafusion/optimizer/src/rewrite_set_comparison.rs b/datafusion/optimizer/src/rewrite_set_comparison.rsnew file mode 100644index 0000000000..c8c35b5187--- /dev/null+++ b/datafusion/optimizer/src/rewrite_set_comparison.rs@@ -0,0 +1,171 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Optimizer rule rewriting `SetComparison` subqueries (e.g. `= ANY`,+//! `> ALL`) into boolean expressions built from `EXISTS` subqueries+//! that capture SQL three-valued logic.++use crate::{OptimizerConfig, OptimizerRule};+use datafusion_common::tree_node::{Transformed, TreeNode};+use datafusion_common::{Column, DFSchema, ExprSchema, Result, ScalarValue, plan_err};+use datafusion_expr::expr::{self, Exists, SetComparison, SetQuantifier};+use datafusion_expr::logical_plan::Subquery;+use datafusion_expr::logical_plan::builder::LogicalPlanBuilder;+use datafusion_expr::{Expr, LogicalPlan, lit};+use std::sync::Arc;++use datafusion_expr::utils::merge_schema;++/// Rewrite `SetComparison` expressions to scalar subqueries that return the+/// correct boolean value (including SQL NULL semantics). After this rule+/// runs, later rules such as `ScalarSubqueryToJoin` can decorrelate and+/// remove the remaining subquery.+#[derive(Debug, Default)]+pub struct RewriteSetComparison;++impl RewriteSetComparison {+ /// Create a new `RewriteSetComparison` optimizer rule.+ pub fn new() -> Self {+ Self+ }++ fn rewrite_plan(&self, plan: LogicalPlan) -> Result<Transformed<LogicalPlan>> {+ let schema = merge_schema(&plan.inputs());+ plan.map_expressions(|expr| {+ expr.transform_up(|expr| rewrite_set_comparison(expr, &schema))+ })+ }+}++impl OptimizerRule for RewriteSetComparison {+ fn name(&self) -> &str {+ "rewrite_set_comparison"+ }++ fn rewrite(+ &self,+ plan: LogicalPlan,+ _config: &dyn OptimizerConfig,+ ) -> Result<Transformed<LogicalPlan>> {+ plan.transform_up_with_subqueries(|plan| self.rewrite_plan(plan))+ }+}++fn rewrite_set_comparison(+ expr: Expr,+ outer_schema: &DFSchema,+) -> Result<Transformed<Expr>> {+ match expr {+ Expr::SetComparison(set_comparison) => {+ let rewritten = build_set_comparison_subquery(set_comparison, outer_schema)?;+ Ok(Transformed::yes(rewritten))+ }+ _ => Ok(Transformed::no(expr)),+ }+}++fn build_set_comparison_subquery(+ set_comparison: SetComparison,+ outer_schema: &DFSchema,+) -> Result<Expr> {+ let SetComparison {+ expr,+ subquery,+ op,+ quantifier,+ } = set_comparison;++ let left_expr = to_outer_reference(*expr, outer_schema)?;+ let subquery_schema = subquery.subquery.schema();+ if subquery_schema.fields().is_empty() {+ return plan_err!("single expression required.");+ }+ // avoid `head_output_expr` for aggr/window plan, it will gives group-by expr if exists+ let right_expr = Expr::Column(Column::from(subquery_schema.qualified_field(0)));++ let comparison = Expr::BinaryExpr(expr::BinaryExpr::new(+ Box::new(left_expr),+ op,+ Box::new(right_expr),+ ));++ let true_exists =+ exists_subquery(&subquery, Expr::IsTrue(Box::new(comparison.clone())))?;+ let null_exists =+ exists_subquery(&subquery, Expr::IsNull(Box::new(comparison.clone())))?;++ let result_expr = match quantifier {+ SetQuantifier::Any => Expr::Case(expr::Case {+ expr: None,+ when_then_expr: vec![+ (Box::new(true_exists), Box::new(lit(true))),+ (+ Box::new(null_exists),+ Box::new(Expr::Literal(ScalarValue::Boolean(None), None)),+ ),+ ],+ else_expr: Some(Box::new(lit(false))),+ }),+ SetQuantifier::All => {+ let false_exists =+ exists_subquery(&subquery, Expr::IsFalse(Box::new(comparison.clone())))?;+ Expr::Case(expr::Case {+ expr: None,+ when_then_expr: vec![+ (Box::new(false_exists), Box::new(lit(false))),+ (+ Box::new(null_exists),+ Box::new(Expr::Literal(ScalarValue::Boolean(None), None)),+ ),+ ],+ else_expr: Some(Box::new(lit(true))),+ })+ }+ };++ Ok(result_expr)+}++fn exists_subquery(subquery: &Subquery, filter: Expr) -> Result<Expr> {+ let plan = LogicalPlanBuilder::from(subquery.subquery.as_ref().clone())+ .filter(filter)?+ .build()?;+ let outer_ref_columns = plan.all_out_ref_exprs();+ Ok(Expr::Exists(Exists {+ subquery: Subquery {+ subquery: Arc::new(plan),+ outer_ref_columns,+ spans: subquery.spans.clone(),+ },+ negated: false,+ }))+}++fn to_outer_reference(expr: Expr, outer_schema: &DFSchema) -> Result<Expr> {+ expr.transform_up(|expr| match expr {+ Expr::Column(col) => {+ let field = outer_schema.field_from_column(&col)?;+ Ok(Transformed::yes(Expr::OuterReferenceColumn(+ Arc::clone(field),+ col,+ )))+ }+ Expr::OuterReferenceColumn(_, _) => Ok(Transformed::no(expr)),+ _ => Ok(Transformed::no(expr)),+ })+ .map(|t| t.data)+}diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rsindex 6d62fbc385..b9ef69dd08 100644--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs@@ -633,6 +633,7 @@ impl ConstEvaluator {| Expr::OuterReferenceColumn(_, _)| Expr::Exists { .. }| Expr::InSubquery(_)+ | Expr::SetComparison(_)| Expr::ScalarSubquery(_)| Expr::WindowFunction { .. }| Expr::GroupingSet(_)diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rsindex 6e4e5d0b6e..9c326a8f6e 100644--- a/datafusion/proto/src/logical_plan/to_proto.rs+++ b/datafusion/proto/src/logical_plan/to_proto.rs@@ -578,6 +578,7 @@ pub fn serialize_expr(Expr::ScalarSubquery(_)| Expr::InSubquery(_)| Expr::Exists { .. }+ | Expr::SetComparison(_)| Expr::OuterReferenceColumn { .. } => {// we would need to add logical plan operators to datafusion.proto to support this// see discussion in https://github.com/apache/datafusion/issues/2565diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.tomlindex a814292a3d..b7338cb764 100644--- a/datafusion/sql/Cargo.toml+++ b/datafusion/sql/Cargo.toml@@ -56,6 +56,7 @@ bigdecimal = { workspace = true }chrono = { workspace = true }datafusion-common = { workspace = true, features = ["sql"] }datafusion-expr = { workspace = true, features = ["sql"] }+datafusion-functions-nested = { workspace = true, features = ["sql"] }indexmap = { workspace = true }log = { workspace = true }recursive = { workspace = true, optional = true }diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rsindex fcd7d6376d..dbf2ce6773 100644--- a/datafusion/sql/src/expr/mod.rs+++ b/datafusion/sql/src/expr/mod.rs@@ -32,6 +32,7 @@ use datafusion_common::{};use datafusion_expr::expr::ScalarFunction;+use datafusion_expr::expr::SetQuantifier;use datafusion_expr::expr::{InList, WildcardOptions};use datafusion_expr::{Between, BinaryExpr, Cast, Expr, ExprSchemable, GetFieldAccess, Like, Literal,@@ -39,6 +40,7 @@ use datafusion_expr::{};use crate::planner::{ContextProvider, PlannerContext, SqlToRel};+use datafusion_functions_nested::expr_fn::array_has;mod binary_op;mod function;@@ -594,32 +596,44 @@ impl<S: ContextProvider> SqlToRel<'_, S> {// ANY/SOME are equivalent, this field specifies which the user// specified but it doesn't affect the plan so ignore the fieldis_some: _,- } => {- let mut binary_expr = RawBinaryExpr {- op: compare_op,- left: self.sql_expr_to_logical_expr(- *left,- schema,- planner_context,- )?,- right: self.sql_expr_to_logical_expr(- *right,- schema,- planner_context,- )?,- };- for planner in self.context_provider.get_expr_planners() {- match planner.plan_any(binary_expr)? {- PlannerResult::Planned(expr) => {- return Ok(expr);- }- PlannerResult::Original(expr) => {- binary_expr = expr;- }+ } => match *right {+ SQLExpr::Subquery(subquery) => self.parse_set_comparison_subquery(+ *left,+ *subquery,+ &compare_op,+ SetQuantifier::Any,+ schema,+ planner_context,+ ),+ _ => {+ if compare_op != BinaryOperator::Eq {+ plan_err!(+ "Unsupported AnyOp: '{compare_op}', only '=' is supported"+ )+ } else {+ let left_expr =+ self.sql_to_expr(*left, schema, planner_context)?;+ let right_expr =+ self.sql_to_expr(*right, schema, planner_context)?;+ Ok(array_has(right_expr, left_expr))}}- not_impl_err!("AnyOp not supported by ExprPlanner: {binary_expr:?}")- }+ },+ SQLExpr::AllOp {+ left,+ compare_op,+ right,+ } => match *right {+ SQLExpr::Subquery(subquery) => self.parse_set_comparison_subquery(+ *left,+ *subquery,+ &compare_op,+ SetQuantifier::All,+ schema,+ planner_context,+ ),+ _ => not_impl_err!("ALL only supports subquery comparison currently"),+ },#[expect(deprecated)]SQLExpr::Wildcard(_token) => Ok(Expr::Wildcard {qualifier: None,diff --git a/datafusion/sql/src/expr/subquery.rs b/datafusion/sql/src/expr/subquery.rsindex ec34ff3d53..6837b2671c 100644--- a/datafusion/sql/src/expr/subquery.rs+++ b/datafusion/sql/src/expr/subquery.rs@@ -17,10 +17,10 @@use crate::planner::{ContextProvider, PlannerContext, SqlToRel};use datafusion_common::{DFSchema, Diagnostic, Result, Span, Spans, plan_err};-use datafusion_expr::expr::{Exists, InSubquery};+use datafusion_expr::expr::{Exists, InSubquery, SetComparison, SetQuantifier};use datafusion_expr::{Expr, LogicalPlan, Subquery};use sqlparser::ast::Expr as SQLExpr;-use sqlparser::ast::{Query, SelectItem, SetExpr};+use sqlparser::ast::{BinaryOperator, Query, SelectItem, SetExpr};use std::sync::Arc;impl<S: ContextProvider> SqlToRel<'_, S> {@@ -162,4 +162,51 @@ impl<S: ContextProvider> SqlToRel<'_, S> {diagnostic.add_help(help_message, None);diagnostic}++ pub(super) fn parse_set_comparison_subquery(+ &self,+ left_expr: SQLExpr,+ subquery: Query,+ compare_op: &BinaryOperator,+ quantifier: SetQuantifier,+ input_schema: &DFSchema,+ planner_context: &mut PlannerContext,+ ) -> Result<Expr> {+ let old_outer_query_schema =+ planner_context.set_outer_query_schema(Some(input_schema.clone().into()));++ let mut spans = Spans::new();+ if let SetExpr::Select(select) = subquery.body.as_ref() {+ for item in &select.projection {+ if let SelectItem::ExprWithAlias { alias, .. } = item+ && let Some(span) = Span::try_from_sqlparser_span(alias.span)+ {+ spans.add_span(span);+ }+ }+ }++ let sub_plan = self.query_to_plan(subquery, planner_context)?;+ let outer_ref_columns = sub_plan.all_out_ref_exprs();+ planner_context.set_outer_query_schema(old_outer_query_schema);++ self.validate_single_column(+ &sub_plan,+ &spans,+ "Too many columns! The subquery should only return one column",+ "Select only one column in the subquery",+ )?;++ let expr_obj = self.sql_to_expr(left_expr, input_schema, planner_context)?;+ Ok(Expr::SetComparison(SetComparison::new(+ Box::new(expr_obj),+ Subquery {+ subquery: Arc::new(sub_plan),+ outer_ref_columns,+ spans,+ },+ self.parse_sql_binary_op(compare_op)?,+ quantifier,+ )))+ }}diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rsindex 5746a568e7..ac7b467920 100644--- a/datafusion/sql/src/unparser/expr.rs+++ b/datafusion/sql/src/unparser/expr.rs@@ -45,7 +45,7 @@ use datafusion_common::{};use datafusion_expr::{Between, BinaryExpr, Case, Cast, Expr, GroupingSet, Like, Operator, TryCast,- expr::{Alias, Exists, InList, ScalarFunction, Sort, WindowFunction},+ expr::{Alias, Exists, InList, ScalarFunction, SetQuantifier, Sort, WindowFunction},};use sqlparser::ast::helpers::attached_token::AttachedToken;use sqlparser::tokenizer::Span;@@ -393,6 +393,33 @@ impl Unparser<'_> {negated: insubq.negated,})}+ Expr::SetComparison(set_cmp) => {+ let left = Box::new(self.expr_to_sql_inner(set_cmp.expr.as_ref())?);+ let sub_statement =+ self.plan_to_sql(set_cmp.subquery.subquery.as_ref())?;+ let sub_query = if let ast::Statement::Query(inner_query) = sub_statement+ {+ inner_query+ } else {+ return plan_err!(+ "Subquery must be a Query, but found {sub_statement:?}"+ );+ };+ let compare_op = self.op_to_sql(&set_cmp.op)?;+ match set_cmp.quantifier {+ SetQuantifier::Any => Ok(ast::Expr::AnyOp {+ left,+ compare_op,+ right: Box::new(ast::Expr::Subquery(sub_query)),+ is_some: false,+ }),+ SetQuantifier::All => Ok(ast::Expr::AllOp {+ left,+ compare_op,+ right: Box::new(ast::Expr::Subquery(sub_query)),+ }),+ }+ }Expr::Exists(Exists { subquery, negated }) => {let sub_statement = self.plan_to_sql(subquery.subquery.as_ref())?;let sub_query = if let ast::Statement::Query(inner_query) = sub_statementdiff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.sltindex 3cedb64895..4d6df4530f 100644--- a/datafusion/sqllogictest/test_files/explain.slt+++ b/datafusion/sqllogictest/test_files/explain.slt@@ -176,6 +176,7 @@ initial_logical_planlogical_plan after resolve_grouping_function SAME TEXT AS ABOVElogical_plan after type_coercion SAME TEXT AS ABOVEanalyzed_logical_plan SAME TEXT AS ABOVE+logical_plan after rewrite_set_comparison SAME TEXT AS ABOVElogical_plan after optimize_unions SAME TEXT AS ABOVElogical_plan after simplify_expressions SAME TEXT AS ABOVElogical_plan after replace_distinct_aggregate SAME TEXT AS ABOVE@@ -197,6 +198,7 @@ logical_plan after single_distinct_aggregation_to_group_by SAME TEXT AS ABOVElogical_plan after eliminate_group_by_constant SAME TEXT AS ABOVElogical_plan after common_sub_expression_eliminate SAME TEXT AS ABOVElogical_plan after optimize_projections TableScan: simple_explain_test projection=[a, b, c]+logical_plan after rewrite_set_comparison SAME TEXT AS ABOVElogical_plan after optimize_unions SAME TEXT AS ABOVElogical_plan after simplify_expressions SAME TEXT AS ABOVElogical_plan after replace_distinct_aggregate SAME TEXT AS ABOVE@@ -535,6 +537,7 @@ initial_logical_planlogical_plan after resolve_grouping_function SAME TEXT AS ABOVElogical_plan after type_coercion SAME TEXT AS ABOVEanalyzed_logical_plan SAME TEXT AS ABOVE+logical_plan after rewrite_set_comparison SAME TEXT AS ABOVElogical_plan after optimize_unions SAME TEXT AS ABOVElogical_plan after simplify_expressions SAME TEXT AS ABOVElogical_plan after replace_distinct_aggregate SAME TEXT AS ABOVE@@ -556,6 +559,7 @@ logical_plan after single_distinct_aggregation_to_group_by SAME TEXT AS ABOVElogical_plan after eliminate_group_by_constant SAME TEXT AS ABOVElogical_plan after common_sub_expression_eliminate SAME TEXT AS ABOVElogical_plan after optimize_projections TableScan: simple_explain_test projection=[a, b, c]+logical_plan after rewrite_set_comparison SAME TEXT AS ABOVElogical_plan after optimize_unions SAME TEXT AS ABOVElogical_plan after simplify_expressions SAME TEXT AS ABOVElogical_plan after replace_distinct_aggregate SAME TEXT AS ABOVEdiff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex da0bfc89d5..e73f4ec3e3 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -430,7 +430,7 @@ SELECT t1_id, t1_name, t1_int, (select t2_id, t2_name FROM t2 WHERE t2.t2_id = t#subquery_not_allowed#In/Exist Subquery is not allowed in ORDER BY clause.-statement error DataFusion error: Invalid \(non-executable\) plan after Analyzer\ncaused by\nError during planning: In/Exist subquery can only be used in Projection, Filter, TableScan, Window functions, Aggregate and Join plan nodes, but was used in \[Sort: t1.t1_int IN \(<subquery>\) ASC NULLS LAST\]+statement error DataFusion error: Invalid \(non-executable\) plan after Analyzer\ncaused by\nError during planning: In/Exist/SetComparison subquery can only be used in Projection, Filter, TableScan, Window functions, Aggregate and Join plan nodes, but was used in \[Sort: t1.t1_int IN \(<subquery>\) ASC NULLS LAST\]SELECT t1_id, t1_name, t1_int FROM t1 order by t1_int in (SELECT t2_int FROM t2 WHERE t1.t1_id > t1.t1_int)#non_aggregated_correlated_scalar_subquery@@ -1469,3 +1469,62 @@ logical_planstatement count 0drop table person;++# Set comparison subqueries (ANY/ALL)+statement ok+create table set_cmp_t(v int) as values (1), (6), (10);++statement ok+create table set_cmp_s(v int) as values (5), (null);++statement ok+create table set_cmp_empty(v int);++query I rowsort+select v from set_cmp_t where v > any(select v from set_cmp_s);+----+10+6++query I rowsort+select v from set_cmp_t where v < all(select v from set_cmp_empty);+----+1+10+6++statement count 0+drop table set_cmp_t;++statement count 0+drop table set_cmp_s;++statement count 0+drop table set_cmp_empty;++query TT+explain select v from (values (1), (6), (10)) set_cmp_t(v) where v > any(select v from (values (5), (null)) set_cmp_s(v));+----+logical_plan+01)Projection: set_cmp_t.v+02)--Filter: __correlated_sq_1.mark OR __correlated_sq_2.mark AND NOT __correlated_sq_3.mark AND Boolean(NULL)+03)----LeftMark Join: Filter: set_cmp_t.v > __correlated_sq_3.v IS TRUE+04)------Filter: __correlated_sq_1.mark OR __correlated_sq_2.mark AND Boolean(NULL)+05)--------LeftMark Join: Filter: set_cmp_t.v > __correlated_sq_2.v IS NULL+06)----------Filter: __correlated_sq_1.mark OR Boolean(NULL)+07)------------LeftMark Join: Filter: set_cmp_t.v > __correlated_sq_1.v IS TRUE+08)--------------SubqueryAlias: set_cmp_t+09)----------------Projection: column1 AS v+10)------------------Values: (Int64(1)), (Int64(6)), (Int64(10))+11)--------------SubqueryAlias: __correlated_sq_1+12)----------------SubqueryAlias: set_cmp_s+13)------------------Projection: column1 AS v+14)--------------------Values: (Int64(5)), (Int64(NULL))+15)----------SubqueryAlias: __correlated_sq_2+16)------------SubqueryAlias: set_cmp_s+17)--------------Projection: column1 AS v+18)----------------Values: (Int64(5)), (Int64(NULL))+19)------SubqueryAlias: __correlated_sq_3+20)--------SubqueryAlias: set_cmp_s+21)----------Projection: column1 AS v+22)------------Values: (Int64(5)), (Int64(NULL))diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/subquery.rs b/datafusion/substrait/src/logical_plan/consumer/expr/subquery.rsindex 15fe7947a1..61a381e9eb 100644--- a/datafusion/substrait/src/logical_plan/consumer/expr/subquery.rs+++ b/datafusion/substrait/src/logical_plan/consumer/expr/subquery.rs@@ -16,12 +16,13 @@// under the License.use crate::logical_plan::consumer::SubstraitConsumer;-use datafusion::common::{DFSchema, Spans, substrait_err};-use datafusion::logical_expr::expr::{Exists, InSubquery};-use datafusion::logical_expr::{Expr, Subquery};+use datafusion::common::{DFSchema, Spans, substrait_datafusion_err, substrait_err};+use datafusion::logical_expr::expr::{Exists, InSubquery, SetComparison, SetQuantifier};+use datafusion::logical_expr::{Expr, Operator, Subquery};use std::sync::Arc;use substrait::proto::expression as substrait_expression;use substrait::proto::expression::subquery::SubqueryType;+use substrait::proto::expression::subquery::set_comparison::{ComparisonOp, ReductionOp};use substrait::proto::expression::subquery::set_predicate::PredicateOp;pub async fn from_subquery(@@ -96,8 +97,53 @@ pub async fn from_subquery(),}}- other_type => {- substrait_err!("Subquery type {other_type:?} not implemented")+ SubqueryType::SetComparison(comparison) => {+ let left = comparison.left.as_ref().ok_or_else(|| {+ substrait_datafusion_err!("SetComparison requires a left expression")+ })?;+ let right = comparison.right.as_ref().ok_or_else(|| {+ substrait_datafusion_err!("SetComparison requires a right relation")+ })?;+ let reduction_op = match ReductionOp::try_from(comparison.reduction_op) {+ Ok(ReductionOp::Any) => SetQuantifier::Any,+ Ok(ReductionOp::All) => SetQuantifier::All,+ _ => {+ return substrait_err!(+ "Unsupported reduction op for SetComparison: {}",+ comparison.reduction_op+ );+ }+ };+ let comparison_op = match ComparisonOp::try_from(comparison.comparison_op)+ {+ Ok(ComparisonOp::Eq) => Operator::Eq,+ Ok(ComparisonOp::Ne) => Operator::NotEq,+ Ok(ComparisonOp::Lt) => Operator::Lt,+ Ok(ComparisonOp::Gt) => Operator::Gt,+ Ok(ComparisonOp::Le) => Operator::LtEq,+ Ok(ComparisonOp::Ge) => Operator::GtEq,+ _ => {+ return substrait_err!(+ "Unsupported comparison op for SetComparison: {}",+ comparison.comparison_op+ );+ }+ };++ let left_expr = consumer.consume_expression(left, input_schema).await?;+ let plan = consumer.consume_rel(right).await?;+ let outer_ref_columns = plan.all_out_ref_exprs();++ Ok(Expr::SetComparison(SetComparison::new(+ Box::new(left_expr),+ Subquery {+ subquery: Arc::new(plan),+ outer_ref_columns,+ spans: Spans::new(),+ },+ comparison_op,+ reduction_op,+ )))}},None => {diff --git a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs b/datafusion/substrait/src/logical_plan/producer/expr/mod.rsindex 5057564d37..74b1a65215 100644--- a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs+++ b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs@@ -141,6 +141,7 @@ pub fn to_substrait_rex(Expr::InList(expr) => producer.handle_in_list(expr, schema),Expr::Exists(expr) => not_impl_err!("Cannot convert {expr:?} to Substrait"),Expr::InSubquery(expr) => producer.handle_in_subquery(expr, schema),+ Expr::SetComparison(expr) => producer.handle_set_comparison(expr, schema),Expr::ScalarSubquery(expr) => {not_impl_err!("Cannot convert {expr:?} to Substrait")}diff --git a/datafusion/substrait/src/logical_plan/producer/expr/subquery.rs b/datafusion/substrait/src/logical_plan/producer/expr/subquery.rsindex f2e6ff5512..e5b9241c10 100644--- a/datafusion/substrait/src/logical_plan/producer/expr/subquery.rs+++ b/datafusion/substrait/src/logical_plan/producer/expr/subquery.rs@@ -16,9 +16,11 @@// under the License.use crate::logical_plan::producer::SubstraitProducer;-use datafusion::common::DFSchemaRef;-use datafusion::logical_expr::expr::InSubquery;+use datafusion::common::{DFSchemaRef, substrait_err};+use datafusion::logical_expr::Operator;+use datafusion::logical_expr::expr::{InSubquery, SetComparison, SetQuantifier};use substrait::proto::expression::subquery::InPredicate;+use substrait::proto::expression::subquery::set_comparison::{ComparisonOp, ReductionOp};use substrait::proto::expression::{RexType, ScalarFunction};use substrait::proto::function_argument::ArgType;use substrait::proto::{Expression, FunctionArgument};@@ -70,3 +72,53 @@ pub fn from_in_subquery(Ok(substrait_subquery)}}++fn comparison_op_to_proto(op: &Operator) -> datafusion::common::Result<ComparisonOp> {+ match op {+ Operator::Eq => Ok(ComparisonOp::Eq),+ Operator::NotEq => Ok(ComparisonOp::Ne),+ Operator::Lt => Ok(ComparisonOp::Lt),+ Operator::Gt => Ok(ComparisonOp::Gt),+ Operator::LtEq => Ok(ComparisonOp::Le),+ Operator::GtEq => Ok(ComparisonOp::Ge),+ _ => substrait_err!("Unsupported operator {op:?} for SetComparison subquery"),+ }+}++fn reduction_op_to_proto(+ quantifier: &SetQuantifier,+) -> datafusion::common::Result<ReductionOp> {+ match quantifier {+ SetQuantifier::Any => Ok(ReductionOp::Any),+ SetQuantifier::All => Ok(ReductionOp::All),+ }+}++pub fn from_set_comparison(+ producer: &mut impl SubstraitProducer,+ set_comparison: &SetComparison,+ schema: &DFSchemaRef,+) -> datafusion::common::Result<Expression> {+ let comparison_op = comparison_op_to_proto(&set_comparison.op)? as i32;+ let reduction_op = reduction_op_to_proto(&set_comparison.quantifier)? as i32;+ let left = producer.handle_expr(set_comparison.expr.as_ref(), schema)?;+ let subquery_plan =+ producer.handle_plan(set_comparison.subquery.subquery.as_ref())?;++ Ok(Expression {+ rex_type: Some(RexType::Subquery(Box::new(+ substrait::proto::expression::Subquery {+ subquery_type: Some(+ substrait::proto::expression::subquery::SubqueryType::SetComparison(+ Box::new(substrait::proto::expression::subquery::SetComparison {+ reduction_op,+ comparison_op,+ left: Some(Box::new(left)),+ right: Some(subquery_plan),+ }),+ ),+ ),+ },+ ))),+ })+}diff --git a/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs b/datafusion/substrait/src/logical_plan/producer/substrait_producer.rsindex ffc920ffe6..c7518bd04e 100644--- a/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs+++ b/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs@@ -20,14 +20,17 @@ use crate::logical_plan::producer::{from_aggregate, from_aggregate_function, from_alias, from_between, from_binary_expr,from_case, from_cast, from_column, from_distinct, from_empty_relation, from_filter,from_in_list, from_in_subquery, from_join, from_like, from_limit, from_literal,- from_projection, from_repartition, from_scalar_function, from_sort,- from_subquery_alias, from_table_scan, from_try_cast, from_unary_expr, from_union,- from_values, from_window, from_window_function, to_substrait_rel, to_substrait_rex,+ from_projection, from_repartition, from_scalar_function, from_set_comparison,+ from_sort, from_subquery_alias, from_table_scan, from_try_cast, from_unary_expr,+ from_union, from_values, from_window, from_window_function, to_substrait_rel,+ to_substrait_rex,};use datafusion::common::{Column, DFSchemaRef, ScalarValue, substrait_err};use datafusion::execution::SessionState;use datafusion::execution::registry::SerializerRegistry;-use datafusion::logical_expr::expr::{Alias, InList, InSubquery, WindowFunction};+use datafusion::logical_expr::expr::{+ Alias, InList, InSubquery, SetComparison, WindowFunction,+};use datafusion::logical_expr::{Aggregate, Between, BinaryExpr, Case, Cast, Distinct, EmptyRelation, Expr, Extension,Filter, Join, Like, Limit, LogicalPlan, Projection, Repartition, Sort, SubqueryAlias,@@ -361,6 +364,14 @@ pub trait SubstraitProducer: Send + Sync + Sized {) -> datafusion::common::Result<Expression> {from_in_subquery(self, in_subquery, schema)}++ fn handle_set_comparison(+ &mut self,+ set_comparison: &SetComparison,+ schema: &DFSchemaRef,+ ) -> datafusion::common::Result<Expression> {+ from_set_comparison(self, set_comparison, schema)+ }}pub struct DefaultSubstraitProducer<'a> {diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rsindex 98b35bf082..f78b255526 100644--- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs@@ -29,14 +29,15 @@ use std::mem::size_of_val;use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema, TimeUnit};use datafusion::common::tree_node::Transformed;-use datafusion::common::{DFSchema, DFSchemaRef, not_impl_err, plan_err};+use datafusion::common::{DFSchema, DFSchemaRef, Spans, not_impl_err, plan_err};use datafusion::error::Result;use datafusion::execution::registry::SerializerRegistry;use datafusion::execution::runtime_env::RuntimeEnv;use datafusion::execution::session_state::SessionStateBuilder;+use datafusion::logical_expr::expr::{SetComparison, SetQuantifier};use datafusion::logical_expr::{- EmptyRelation, Extension, InvariantLevel, LogicalPlan, PartitionEvaluator,- Repartition, UserDefinedLogicalNode, Values, Volatility,+ EmptyRelation, Extension, InvariantLevel, LogicalPlan, Operator, PartitionEvaluator,+ Repartition, Subquery, UserDefinedLogicalNode, Values, Volatility,};use datafusion::optimizer::simplify_expressions::expr_simplifier::THRESHOLD_INLINE_INLIST;use datafusion::prelude::*;@@ -689,6 +690,29 @@ async fn roundtrip_exists_filter() -> Result<()> {Ok(())}+// assemble logical plan manually to ensure SetComparison expr is present (not rewrite away)+#[tokio::test]+async fn roundtrip_set_comparison_any_substrait() -> Result<()> {+ let ctx = create_context().await?;+ let plan = build_set_comparison_plan(&ctx, SetQuantifier::Any, Operator::Gt).await?;+ let proto = to_substrait_plan(&plan, &ctx.state())?;+ let roundtrip_plan = from_substrait_plan(&ctx.state(), &proto).await?;+ assert_set_comparison_predicate(&roundtrip_plan, Operator::Gt, SetQuantifier::Any);+ Ok(())+}++// assemble logical plan manually to ensure SetComparison expr is present (not rewrite away)+#[tokio::test]+async fn roundtrip_set_comparison_all_substrait() -> Result<()> {+ let ctx = create_context().await?;+ let plan =+ build_set_comparison_plan(&ctx, SetQuantifier::All, Operator::NotEq).await?;+ let proto = to_substrait_plan(&plan, &ctx.state())?;+ let roundtrip_plan = from_substrait_plan(&ctx.state(), &proto).await?;+ assert_set_comparison_predicate(&roundtrip_plan, Operator::NotEq, SetQuantifier::All);+ Ok(())+}+#[tokio::test]async fn roundtrip_not_exists_filter_left_anti_join() -> Result<()> {let plan = generate_plan_from_sql(@@ -1865,6 +1889,56 @@ async fn assert_substrait_sql(substrait_plan: Plan, sql: &str) -> Result<()> {Ok(())}+async fn build_set_comparison_plan(+ ctx: &SessionContext,+ quantifier: SetQuantifier,+ op: Operator,+) -> Result<LogicalPlan> {+ let base_scan = ctx.table("data").await?.into_unoptimized_plan();+ let subquery_scan = ctx.table("data2").await?.into_unoptimized_plan();+ let subquery_plan = LogicalPlanBuilder::from(subquery_scan)+ .project(vec![col("data2.a")])?+ .build()?;+ let predicate = Expr::SetComparison(SetComparison::new(+ Box::new(col("data.a")),+ Subquery {+ subquery: Arc::new(subquery_plan),+ outer_ref_columns: vec![],+ spans: Spans::new(),+ },+ op,+ quantifier,+ ));++ LogicalPlanBuilder::from(base_scan)+ .filter(predicate)?+ .project(vec![col("data.a")])?+ .build()+}++fn assert_set_comparison_predicate(+ plan: &LogicalPlan,+ expected_op: Operator,+ expected_quantifier: SetQuantifier,+) {+ let predicate = match plan {+ LogicalPlan::Projection(p) => match p.input.as_ref() {+ LogicalPlan::Filter(filter) => &filter.predicate,+ other => panic!("expected Filter inside Projection, got {other:?}"),+ },+ LogicalPlan::Filter(filter) => &filter.predicate,+ other => panic!("expected Filter plan, got {other:?}"),+ };++ match predicate {+ Expr::SetComparison(set_comparison) => {+ assert_eq!(set_comparison.op, expected_op);+ assert_eq!(set_comparison.quantifier, expected_quantifier);+ }+ other => panic!("expected SetComparison predicate, got {other:?}"),+ }+}+async fn roundtrip_fill_na(sql: &str) -> Result<()> {let ctx = create_context().await?;let df = ctx.sql(sql).await?;