Apache DataFusion SQL Query Engine
fix: apply the left side schema on the right side in set expressions (#21052)
Marko Grujic8cf70ec76c6f
diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rsindex 5381313e2e..188daa724c 100644--- a/datafusion/expr/src/logical_plan/builder.rs+++ b/datafusion/expr/src/logical_plan/builder.rs@@ -593,7 +593,23 @@ impl LogicalPlanBuilder {self,expr: Vec<(impl Into<SelectExpr>, bool)>,) -> Result<Self> {- project_with_validation(Arc::unwrap_or_clone(self.plan), expr).map(Self::new)+ project_with_validation(Arc::unwrap_or_clone(self.plan), expr, None)+ .map(Self::new)+ }++ /// Apply a projection, aliasing non-Column/non-Alias expressions to+ /// match the field names from the provided schema.+ pub fn project_with_validation_and_schema(+ self,+ expr: impl IntoIterator<Item = impl Into<SelectExpr>>,+ schema: &DFSchemaRef,+ ) -> Result<Self> {+ project_with_validation(+ Arc::unwrap_or_clone(self.plan),+ expr.into_iter().map(|e| (e, true)),+ Some(schema),+ )+ .map(Self::new)}/// Select the given column indices@@ -1916,7 +1932,7 @@ pub fn project(plan: LogicalPlan,expr: impl IntoIterator<Item = impl Into<SelectExpr>>,) -> Result<LogicalPlan> {- project_with_validation(plan, expr.into_iter().map(|e| (e, true)))+ project_with_validation(plan, expr.into_iter().map(|e| (e, true)), None)}/// Create Projection. Similar to project except that the expressions@@ -1929,6 +1945,7 @@ pub fn project(fn project_with_validation(plan: LogicalPlan,expr: impl IntoIterator<Item = (impl Into<SelectExpr>, bool)>,+ schema: Option<&DFSchemaRef>,) -> Result<LogicalPlan> {let mut projected_expr = vec![];let mut has_wildcard = false;@@ -1987,12 +2004,24 @@ fn project_with_validation(}}}+if has_wildcard && projected_expr.is_empty() && !plan.schema().fields().is_empty() {return plan_err!("SELECT list is empty after resolving * expressions, \the wildcard expanded to zero columns");}++ // When inside a set expression, alias non-Column/non-Alias expressions+ // to match the left side's field names, avoiding duplicate name errors.+ if let Some(schema) = &schema {+ for (expr, field) in projected_expr.iter_mut().zip(schema.fields()) {+ if !matches!(expr, Expr::Column(_) | Expr::Alias(_)) {+ *expr = std::mem::take(expr).alias(field.name());+ }+ }+ }+validate_unique_names("Projections", projected_expr.iter())?;Projection::try_new(projected_expr, Arc::new(plan)).map(LogicalPlan::Projection)diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rsindex b7e270e4f0..32daf65a71 100644--- a/datafusion/sql/src/planner.rs+++ b/datafusion/sql/src/planner.rs@@ -270,6 +270,10 @@ pub struct PlannerContext {outer_from_schema: Option<DFSchemaRef>,/// The query schema defined by the tablecreate_table_schema: Option<DFSchemaRef>,+ /// When planning non-first queries in a set expression+ /// (UNION/INTERSECT/EXCEPT), holds the schema of the left-most query.+ /// Used to alias duplicate expressions to match the left side's field names.+ set_expr_left_schema: Option<DFSchemaRef>,}impl Default for PlannerContext {@@ -287,6 +291,7 @@ impl PlannerContext {outer_queries_schemas_stack: vec![],outer_from_schema: None,create_table_schema: None,+ set_expr_left_schema: None,}}@@ -400,6 +405,14 @@ impl PlannerContext {pub(super) fn remove_cte(&mut self, cte_name: &str) {self.ctes.remove(cte_name);}++ /// Sets the left-most set expression schema, returning the previous value+ pub(super) fn set_set_expr_left_schema(+ &mut self,+ schema: Option<DFSchemaRef>,+ ) -> Option<DFSchemaRef> {+ std::mem::replace(&mut self.set_expr_left_schema, schema)+ }}/// SQL query planner and binderdiff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rsindex 1b7bb856a5..e320d2ee6e 100644--- a/datafusion/sql/src/query.rs+++ b/datafusion/sql/src/query.rs@@ -152,7 +152,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {let empty_from = matches!(plan, LogicalPlan::EmptyRelation(_));let select_exprs =self.prepare_select_exprs(&plan, exprs, empty_from, planner_context)?;- self.project(plan, select_exprs)+ self.project(plan, select_exprs, None)}PipeOperator::Extend { exprs } => {let empty_from = matches!(plan, LogicalPlan::EmptyRelation(_));@@ -162,7 +162,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {std::iter::once(SelectExpr::Wildcard(WildcardOptions::default())).chain(extend_exprs).collect();- self.project(plan, all_exprs)+ self.project(plan, all_exprs, None)}PipeOperator::As { alias } => self.apply_table_alias(plan,diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rsindex 1b35070b29..82cca91d4e 100644--- a/datafusion/sql/src/select.rs+++ b/datafusion/sql/src/select.rs@@ -29,7 +29,7 @@ use crate::utils::{use datafusion_common::error::DataFusionErrorBuilder;use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};-use datafusion_common::{Column, DFSchema, Result, not_impl_err, plan_err};+use datafusion_common::{Column, DFSchema, DFSchemaRef, Result, not_impl_err, plan_err};use datafusion_common::{RecursionUnnestOption, UnnestOptions};use datafusion_expr::expr::{PlannedReplaceSelectItem, WildcardOptions};use datafusion_expr::expr_rewriter::{@@ -90,6 +90,10 @@ impl<S: ContextProvider> SqlToRel<'_, S> {return not_impl_err!("SORT BY");}+ // Capture and clear set expression schema so it doesn't leak+ // into subqueries planned during FROM clause handling.+ let set_expr_left_schema = planner_context.set_set_expr_left_schema(None);+// Process `from` clauselet plan = self.plan_from_tables(select.from, planner_context)?;let empty_from = matches!(plan, LogicalPlan::EmptyRelation(_));@@ -110,7 +114,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {)?;// Having and group by clause may reference aliases defined in select projection- let projected_plan = self.project(base_plan.clone(), select_exprs)?;+ let projected_plan =+ self.project(base_plan.clone(), select_exprs, set_expr_left_schema)?;let select_exprs = projected_plan.expressions();let order_by =@@ -895,18 +900,29 @@ impl<S: ContextProvider> SqlToRel<'_, S> {&self,input: LogicalPlan,expr: Vec<SelectExpr>,+ set_expr_left_schema: Option<DFSchemaRef>,) -> Result<LogicalPlan> {// convert to Expr for validate_schema_satisfies_exprs- let exprs = expr+ let plain_exprs = expr.iter().filter_map(|e| match e {SelectExpr::Expression(expr) => Some(expr.to_owned()),_ => None,}).collect::<Vec<_>>();- self.validate_schema_satisfies_exprs(input.schema(), &exprs)?;-- LogicalPlanBuilder::from(input).project(expr)?.build()+ self.validate_schema_satisfies_exprs(input.schema(), &plain_exprs)?;++ // When inside a set expression, pass the left-most schema so+ // that expressions get aliased to match, avoiding duplicate+ // name errors from expressions like `count(*), count(*)`.+ let builder = LogicalPlanBuilder::from(input);+ if let Some(left_schema) = set_expr_left_schema {+ builder+ .project_with_validation_and_schema(expr, &left_schema)?+ .build()+ } else {+ builder.project(expr)?.build()+ }}/// Create an aggregate plan.diff --git a/datafusion/sql/src/set_expr.rs b/datafusion/sql/src/set_expr.rsindex d4e771cb48..dc8e4f14d1 100644--- a/datafusion/sql/src/set_expr.rs+++ b/datafusion/sql/src/set_expr.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+use std::sync::Arc;+use crate::planner::{ContextProvider, PlannerContext, SqlToRel};use datafusion_common::{DataFusionError, Diagnostic, Result, Span, not_impl_err, plan_err,@@ -42,7 +44,23 @@ impl<S: ContextProvider> SqlToRel<'_, S> {let left_span = Span::try_from_sqlparser_span(left.span());let right_span = Span::try_from_sqlparser_span(right.span());let left_plan = self.set_expr_to_plan(*left, planner_context);+ // Store the left plan's schema so that the right side can+ // alias duplicate expressions to match. Skip for BY NAME+ // operations since those match columns by name, not position.+ if let Ok(plan) = &left_plan+ && plan.schema().fields().len() > 1+ && !matches!(+ set_quantifier,+ SetQuantifier::ByName+ | SetQuantifier::AllByName+ | SetQuantifier::DistinctByName+ )+ {+ planner_context+ .set_set_expr_left_schema(Some(Arc::clone(plan.schema())));+ }let right_plan = self.set_expr_to_plan(*right, planner_context);+ planner_context.set_set_expr_left_schema(None);let (left_plan, right_plan) = match (left_plan, right_plan) {(Ok(left_plan), Ok(right_plan)) => (left_plan, right_plan),(Err(left_err), Err(right_err)) => {diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rsindex fd606af3a6..2c2c7eac8b 100644--- a/datafusion/sql/tests/sql_integration.rs+++ b/datafusion/sql/tests/sql_integration.rs@@ -2654,6 +2654,106 @@ fn union_all_by_name_same_column_names() {);}+#[test]+fn union_all_with_duplicate_expressions() {+ let sql = "\+ SELECT 0 a, 0 b \+ UNION ALL SELECT 1, 1 \+ UNION ALL SELECT count(*), count(*) FROM orders";+ let plan = logical_plan(sql).unwrap();+ assert_snapshot!(+ plan,+ @r"+ Union+ Union+ Projection: Int64(0) AS a, Int64(0) AS b+ EmptyRelation: rows=1+ Projection: Int64(1) AS a, Int64(1) AS b+ EmptyRelation: rows=1+ Projection: count(*) AS a, count(*) AS b+ Aggregate: groupBy=[[]], aggr=[[count(*)]]+ TableScan: orders+ "+ );+}++#[test]+fn union_with_qualified_and_duplicate_expressions() {+ let sql = "\+ SELECT 0 a, id b, price c, 0 d FROM test_decimal \+ UNION SELECT 1, *, 1 FROM test_decimal";+ let plan = logical_plan(sql).unwrap();+ assert_snapshot!(+ plan,+ @"+ Distinct:+ Union+ Projection: Int64(0) AS a, test_decimal.id AS b, test_decimal.price AS c, Int64(0) AS d+ TableScan: test_decimal+ Projection: Int64(1) AS a, test_decimal.id, test_decimal.price, Int64(1) AS d+ TableScan: test_decimal+ "+ );+}++#[test]+fn intersect_with_duplicate_expressions() {+ let sql = "\+ SELECT 0 a, 0 b \+ INTERSECT SELECT 1, 1 \+ INTERSECT SELECT count(*), count(*) FROM orders";+ let plan = logical_plan(sql).unwrap();+ assert_snapshot!(+ plan,+ @r"+ LeftSemi Join: left.a = right.a, left.b = right.b+ Distinct:+ SubqueryAlias: left+ LeftSemi Join: left.a = right.a, left.b = right.b+ Distinct:+ SubqueryAlias: left+ Projection: Int64(0) AS a, Int64(0) AS b+ EmptyRelation: rows=1+ SubqueryAlias: right+ Projection: Int64(1) AS a, Int64(1) AS b+ EmptyRelation: rows=1+ SubqueryAlias: right+ Projection: count(*) AS a, count(*) AS b+ Aggregate: groupBy=[[]], aggr=[[count(*)]]+ TableScan: orders+ "+ );+}++#[test]+fn except_with_duplicate_expressions() {+ let sql = "\+ SELECT 0 a, 0 b \+ EXCEPT SELECT 1, 1 \+ EXCEPT SELECT count(*), count(*) FROM orders";+ let plan = logical_plan(sql).unwrap();+ assert_snapshot!(+ plan,+ @r"+ LeftAnti Join: left.a = right.a, left.b = right.b+ Distinct:+ SubqueryAlias: left+ LeftAnti Join: left.a = right.a, left.b = right.b+ Distinct:+ SubqueryAlias: left+ Projection: Int64(0) AS a, Int64(0) AS b+ EmptyRelation: rows=1+ SubqueryAlias: right+ Projection: Int64(1) AS a, Int64(1) AS b+ EmptyRelation: rows=1+ SubqueryAlias: right+ Projection: count(*) AS a, count(*) AS b+ Aggregate: groupBy=[[]], aggr=[[count(*)]]+ TableScan: orders+ "+ );+}+#[test]fn empty_over() {let sql = "SELECT order_id, MAX(order_id) OVER () from orders";diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.sltindex 0f4b02ae25..3871468411 100644--- a/datafusion/sqllogictest/test_files/union.slt+++ b/datafusion/sqllogictest/test_files/union.slt@@ -256,6 +256,30 @@ Bob_newJohnJohn_new+# Test UNION ALL with unaliased duplicate literal values on the right side.+# The second projection will inherit field names from the first one, and so+# pass the unique projection expression name check.+query TII rowsort+SELECT name, 1 as table, 1 as row FROM t1 WHERE id = 1+UNION ALL+SELECT name, 2, 2 FROM t2 WHERE id = 2+----+Alex 1 1+Bob 2 2++# Test nested UNION, EXCEPT, INTERSECT with duplicate unaliased literals.+# Only the first SELECT has column aliases, which should propagate to all projections.+query III rowsort+SELECT 1 as a, 0 as b, 0 as c+UNION ALL+((SELECT 2, 0, 0 UNION ALL SELECT 3, 0, 0) EXCEPT SELECT 3, 0, 0)+UNION ALL+(SELECT 4, 0, 0 INTERSECT SELECT 4, 0, 0)+----+1 0 0+2 0 0+4 0 0+# Plan is unnestedquery TTEXPLAIN SELECT name FROM t1 UNION ALL (SELECT name from t2 UNION ALL SELECT name || '_new' from t2)