Apache DataFusion SQL Query Engine
Fix DuckDB unparse for optimized join projections (#23002)
Goutam Adwant48a5e21263d0
diff --git a/datafusion/core/tests/sql/unparser.rs b/datafusion/core/tests/sql/unparser.rsindex d6ca872e19..7762512fac 100644--- a/datafusion/core/tests/sql/unparser.rs+++ b/datafusion/core/tests/sql/unparser.rs@@ -37,14 +37,19 @@use std::fs::ReadDir;use std::future::Future;+use std::sync::Arc;use arrow::array::RecordBatch;+use arrow::datatypes::{DataType, Field, Schema};use datafusion::common::Result;+use datafusion::datasource::empty::EmptyTable;use datafusion::prelude::{ParquetReadOptions, SessionContext};+use datafusion_catalog::memory::MemorySchemaProvider;+use datafusion_catalog::{CatalogProvider, MemoryCatalogProvider, SchemaProvider};use datafusion_common::Column;use datafusion_expr::Expr;use datafusion_sql::unparser::Unparser;-use datafusion_sql::unparser::dialect::DefaultDialect;+use datafusion_sql::unparser::dialect::{DefaultDialect, DuckDBDialect};use itertools::Itertools;use recursive::{set_minimum_stack_size, set_stack_allocation_size};@@ -218,6 +223,126 @@ async fn sort_batches(df.collect().await}+const ISSUE_22961_QUERY: &str = r#"+SELECT * FROM+(+SELECT+ item_id,+ order_id,+ product_id,+ quantity,+ unit_price,+ quantity * unit_price AS line_total+ FROM+ "warehouse"."main"."order_items"+) oi+JOIN (+ SELECT+ order_id,+ customer_id,+ order_date,+ lower(STATUS) AS STATUS,+ lower(channel) AS channel,+ coalesce(discount_pct, 0) AS discount_pct,+ coalesce(shipping_cost, 0) AS shipping_cost,+ STATUS IN ('completed', 'shipped') AS is_fulfilled+ FROM+ "warehouse"."main"."orders"+) o USING (order_id)+JOIN (+ SELECT+ p.product_id,+ p.category_id,+ p.sku,+ p.name AS product_name,+ p.price,+ p.cost,+ p.weight_kg,+ p.is_active,+ p.stock_qty,+ round(p.price - p.cost, 2) AS gross_margin,+ round((p.price - p.cost) / nullif(p.price, 0), 4) AS margin_pct,+ c.name AS category_name+ FROM+ "warehouse"."main"."products" p+ LEFT JOIN "warehouse"."main"."categories" c USING (category_id)+) p USING (product_id)+"#;++fn issue_22961_context() -> Result<SessionContext> {+ let ctx = SessionContext::new();++ let schema_provider = Arc::new(MemorySchemaProvider::new());+ schema_provider.register_table(+ "order_items".to_string(),+ Arc::new(EmptyTable::new(Arc::new(Schema::new(vec![+ Field::new("item_id", DataType::Int32, false),+ Field::new("order_id", DataType::Int32, true),+ Field::new("product_id", DataType::Int32, true),+ Field::new("quantity", DataType::Int32, true),+ Field::new("unit_price", DataType::Decimal128(10, 2), true),+ ])))),+ )?;+ schema_provider.register_table(+ "orders".to_string(),+ Arc::new(EmptyTable::new(Arc::new(Schema::new(vec![+ Field::new("order_id", DataType::Int32, false),+ Field::new("customer_id", DataType::Int32, true),+ Field::new("order_date", DataType::Date32, true),+ Field::new("status", DataType::Utf8, true),+ Field::new("channel", DataType::Utf8, true),+ Field::new("discount_pct", DataType::Decimal128(5, 2), true),+ Field::new("shipping_cost", DataType::Decimal128(8, 2), true),+ ])))),+ )?;+ schema_provider.register_table(+ "products".to_string(),+ Arc::new(EmptyTable::new(Arc::new(Schema::new(vec![+ Field::new("product_id", DataType::Int32, false),+ Field::new("category_id", DataType::Int32, true),+ Field::new("sku", DataType::Utf8, true),+ Field::new("name", DataType::Utf8, true),+ Field::new("price", DataType::Decimal128(10, 2), true),+ Field::new("cost", DataType::Decimal128(10, 2), true),+ Field::new("weight_kg", DataType::Decimal128(6, 3), true),+ Field::new("is_active", DataType::Boolean, true),+ Field::new("stock_qty", DataType::Int32, true),+ ])))),+ )?;+ schema_provider.register_table(+ "categories".to_string(),+ Arc::new(EmptyTable::new(Arc::new(Schema::new(vec![+ Field::new("category_id", DataType::Int32, false),+ Field::new("name", DataType::Utf8, true),+ Field::new("parent_id", DataType::Int32, true),+ Field::new("display_rank", DataType::Int32, true),+ ])))),+ )?;++ let catalog = Arc::new(MemoryCatalogProvider::new());+ catalog.register_schema("main", schema_provider)?;+ ctx.register_catalog("warehouse", catalog);++ Ok(ctx)+}++#[tokio::test]+async fn optimized_duckdb_unparse_preserves_derived_table_scope() -> Result<()> {+ let ctx = issue_22961_context()?;+ let plan = ctx.sql(ISSUE_22961_QUERY).await?.into_optimized_plan()?;+ let dialect = DuckDBDialect::new();+ let unparser = Unparser::new(&dialect);+ let sql = unparser.plan_to_sql(&plan)?.to_string();++ assert!(!sql.contains(r#""o"."__common_expr_1""#));+ assert!(!sql.contains(r#""o"."__common_expr_2""#));+ assert!(sql.contains(+ r#"ON "oi"."order_id" = "o"."order_id" INNER JOIN (SELECT "p"."product_id""#+ ));++ Ok(())+}+/// The outcome of running a single roundtrip test.////// A successful test produces [`TestCaseResult::Success`].diff --git a/datafusion/sql/src/unparser/ast.rs b/datafusion/sql/src/unparser/ast.rsindex 4b4e56c40c..7418d0b5b7 100644--- a/datafusion/sql/src/unparser/ast.rs+++ b/datafusion/sql/src/unparser/ast.rs@@ -264,6 +264,9 @@ impl SelectBuilder {pub fn pop_from(&mut self) -> Option<TableWithJoinsBuilder> {self.from.pop()}+ pub fn has_selection(&self) -> bool {+ self.selection.is_some()+ }pub fn lateral_views(&mut self, value: Vec<ast::LateralView>) -> &mut Self {self.lateral_views = value;self@@ -483,6 +486,7 @@ pub struct RelationBuilder {enum TableFactorBuilder {Table(TableRelationBuilder),Derived(DerivedRelationBuilder),+ NestedJoin(ast::TableWithJoins, Option<ast::TableAlias>),Unnest(UnnestRelationBuilder),Flatten(FlattenRelationBuilder),Empty,@@ -501,6 +505,15 @@ impl RelationBuilder {self}+ pub fn nested_join(+ &mut self,+ value: ast::TableWithJoins,+ alias: Option<ast::TableAlias>,+ ) -> &mut Self {+ self.relation = Some(TableFactorBuilder::NestedJoin(value, alias));+ self+ }+pub fn unnest(&mut self, value: UnnestRelationBuilder) -> &mut Self {self.relation = Some(TableFactorBuilder::Unnest(value));self@@ -524,6 +537,9 @@ impl RelationBuilder {Some(TableFactorBuilder::Derived(ref mut rel_builder)) => {rel_builder.alias = value;}+ Some(TableFactorBuilder::NestedJoin(_, ref mut alias)) => {+ *alias = value;+ }Some(TableFactorBuilder::Unnest(ref mut rel_builder)) => {rel_builder.alias = value;}@@ -539,6 +555,12 @@ impl RelationBuilder {Ok(match self.relation {Some(TableFactorBuilder::Table(ref value)) => Some(value.build()?),Some(TableFactorBuilder::Derived(ref value)) => Some(value.build()?),+ Some(TableFactorBuilder::NestedJoin(ref table_with_joins, ref alias)) => {+ Some(ast::TableFactor::NestedJoin {+ table_with_joins: Box::new(table_with_joins.clone()),+ alias: alias.clone(),+ })+ }Some(TableFactorBuilder::Unnest(ref value)) => Some(value.build()?),Some(TableFactorBuilder::Flatten(ref value)) => Some(value.build()?),Some(TableFactorBuilder::Empty) => None,diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rsindex 861de01e75..37ff8145d3 100644--- a/datafusion/sql/src/unparser/plan.rs+++ b/datafusion/sql/src/unparser/plan.rs@@ -42,8 +42,8 @@ use crate::unparser::{};use crate::utils::UNNEST_PLACEHOLDER;use datafusion_common::{- Column, DataFusionError, Result, ScalarValue, TableReference, assert_or_internal_err,- internal_datafusion_err, internal_err, not_impl_err,+ Column, DFSchema, DataFusionError, Result, ScalarValue, TableReference,+ assert_or_internal_err, internal_datafusion_err, internal_err, not_impl_err,tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion},utils::combine_limit,};@@ -571,8 +571,9 @@ impl Unparser<'_> {let input_schema = window.input.schema();let mut alias_rewriter = TableAliasRewriter {- table_schema: input_schema.as_arrow(),+ table_schema: input_schema.as_ref(),alias_name: TableReference::bare(input_alias),+ rewrite_unqualified: true,};let window_expr = window.window_expr@@ -1160,6 +1161,11 @@ impl Unparser<'_> {}None => Arc::clone(left_plan),};+ let left_plan = if already_projected {+ Self::unwrap_qualified_passthrough_join_projection(left_plan)+ } else {+ left_plan+ };self.select_to_sql_recursively(left_plan.as_ref(),@@ -1185,13 +1191,22 @@ impl Unparser<'_> {};let mut right_relation = RelationBuilder::default();-- self.select_to_sql_recursively(- right_plan.as_ref(),- query,- select,- &mut right_relation,- )?;+ if already_projected+ && let Some(nested_relation) = self+ .qualified_passthrough_join_projection_to_nested_relation(+ right_plan.as_ref(),+ query,+ )?+ {+ right_relation = nested_relation;+ } else {+ self.select_to_sql_recursively(+ right_plan.as_ref(),+ query,+ select,+ &mut right_relation,+ )?;+ }let (join_filters, where_filters) = Self::split_join_on_and_where_filters(join.join_type,@@ -1910,6 +1925,68 @@ impl Unparser<'_> {)}+ fn is_qualified_passthrough_projection(projection: &Projection) -> bool {+ projection+ .expr+ .iter()+ .all(|expr| matches!(expr, Expr::Column(column) if column.relation.is_some()))+ }++ fn unwrap_qualified_passthrough_join_projection(+ plan: Arc<LogicalPlan>,+ ) -> Arc<LogicalPlan> {+ if let LogicalPlan::Projection(projection) = plan.as_ref()+ && matches!(projection.input.as_ref(), LogicalPlan::Join(_))+ && Self::is_qualified_passthrough_projection(projection)+ {+ Arc::clone(&projection.input)+ } else {+ plan+ }+ }++ fn qualified_passthrough_join_projection_to_nested_relation(+ &self,+ plan: &LogicalPlan,+ query: &mut Option<QueryBuilder>,+ ) -> Result<Option<RelationBuilder>> {+ let LogicalPlan::Projection(projection) = plan else {+ return Ok(None);+ };+ if !matches!(projection.input.as_ref(), LogicalPlan::Join(_))+ || !Self::is_qualified_passthrough_projection(projection)+ {+ return Ok(None);+ }++ let original_query = query.clone();+ let mut nested_select = SelectBuilder::default();+ nested_select.push_from(TableWithJoinsBuilder::default());+ let mut nested_relation = RelationBuilder::default();+ self.select_to_sql_recursively(+ projection.input.as_ref(),+ query,+ &mut nested_select,+ &mut nested_relation,+ )?;+ if nested_select.has_selection() {+ *query = original_query;+ return Ok(None);+ }++ let Some(mut nested_from) = nested_select.pop_from() else {+ return internal_err!("Failed to build nested join relation");+ };+ nested_from.relation(nested_relation);+ let Some(table_with_joins) = nested_from.build()? else {+ return internal_err!("Failed to build nested join relation");+ };++ let mut relation = RelationBuilder::default();+ relation.nested_join(table_with_joins, None);+ Ok(Some(relation))+ }+/// Try to unparse a table scan with pushdown operations into a new subquery plan./// If the table scan is without any pushdown operations, return None.fn unparse_table_scan_pushdown(@@ -1924,10 +2001,15 @@ impl Unparser<'_> {return Ok(None);}let table_schema = table_scan.source.schema();+ let filter_schema = DFSchema::try_from_qualified_schema(+ table_scan.table_name.clone(),+ table_schema.as_ref(),+ )?;let mut filter_alias_rewriter =alias.as_ref().map(|alias_name| TableAliasRewriter {- table_schema: &table_schema,+ table_schema: &filter_schema,alias_name: alias_name.clone(),+ rewrite_unqualified: true,});let mut builder = LogicalPlanBuilder::scan(@@ -2037,8 +2119,9 @@ impl Unparser<'_> {let exprs = if alias.is_some() {let mut alias_rewriter =alias.as_ref().map(|alias_name| TableAliasRewriter {- table_schema: plan.schema().as_arrow(),+ table_schema: plan.schema().as_ref(),alias_name: alias_name.clone(),+ rewrite_unqualified: false,});projection.exprdiff --git a/datafusion/sql/src/unparser/rewrite.rs b/datafusion/sql/src/unparser/rewrite.rsindex a6bfba4cca..6ee66f6193 100644--- a/datafusion/sql/src/unparser/rewrite.rs+++ b/datafusion/sql/src/unparser/rewrite.rs@@ -17,10 +17,9 @@use std::{collections::HashSet, sync::Arc};-use arrow::datatypes::Schema;use datafusion_common::tree_node::TreeNodeContainer;use datafusion_common::{- Column, HashMap, Result, TableReference,+ Column, DFSchema, HashMap, Result, TableReference,tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRewriter},};use datafusion_expr::expr::{Alias, UNNEST_COLUMN_PREFIX};@@ -502,20 +501,24 @@ fn find_projection(logical_plan: &LogicalPlan) -> Option<&Projection> {}/// A `TreeNodeRewriter` implementation that rewrites `Expr::Column` expressions by-/// replacing the column's name with an alias if the column exists in the provided schema.+/// replacing the column's qualifier with an alias if the column resolves to a+/// qualified field in the provided schema.////// This is typically used to apply table aliases in query plans, ensuring that/// the column references in the expressions use the correct table alias.////// # Fields///-/// * `table_schema`: The schema (`SchemaRef`) representing the table structure-/// from which the columns are referenced. This is used to look up columns by their names.+/// * `table_schema`: The schema representing the table structure from which the+/// columns are referenced. This is used to look up columns by their names and qualifiers./// * `alias_name`: The alias (`TableReference`) that will replace the table name/// in the column references when applicable.+/// * `rewrite_unqualified`: Whether columns that resolve to unqualified fields+/// in `table_schema` should also be rewritten to `alias_name`.pub struct TableAliasRewriter<'a> {- pub table_schema: &'a Schema,+ pub table_schema: &'a DFSchema,pub alias_name: TableReference,+ pub rewrite_unqualified: bool,}impl TreeNodeRewriter for TableAliasRewriter<'_> {@@ -524,12 +527,23 @@ impl TreeNodeRewriter for TableAliasRewriter<'_> {fn f_down(&mut self, expr: Expr) -> Result<Transformed<Expr>> {match expr {Expr::Column(column) => {- if let Ok(field) = self.table_schema.field_with_name(&column.name) {- let new_column =- Column::new(Some(self.alias_name.clone()), field.name().clone());- Ok(Transformed::yes(Expr::Column(new_column)))- } else {- Ok(Transformed::no(Expr::Column(column)))+ match self+ .table_schema+ .qualified_field_from_column(&column)+ .or_else(|_| {+ self.table_schema+ .qualified_field_with_unqualified_name(&column.name)+ }) {+ Ok((qualifier, field))+ if qualifier.is_some() || self.rewrite_unqualified =>+ {+ let new_column = Column::new(+ Some(self.alias_name.clone()),+ field.name().clone(),+ );+ Ok(Transformed::yes(Expr::Column(new_column)))+ }+ Ok(_) | Err(_) => Ok(Transformed::no(Expr::Column(column))),}}_ => Ok(Transformed::no(expr)),@@ -540,7 +554,7 @@ impl TreeNodeRewriter for TableAliasRewriter<'_> {#[cfg(test)]mod tests {use super::*;- use arrow::datatypes::{DataType, Field};+ use arrow::datatypes::{DataType, Field, Schema};use datafusion_expr::{LogicalPlanBuilder, col, table_scan};// this is a regression test: when the outer projection has fewer expressions thandiff --git a/datafusion/sql/src/unparser/utils.rs b/datafusion/sql/src/unparser/utils.rsindex 1cc023d112..949b49eb77 100644--- a/datafusion/sql/src/unparser/utils.rs+++ b/datafusion/sql/src/unparser/utils.rs@@ -22,7 +22,7 @@ use super::{rewrite::TableAliasRewriter,};use datafusion_common::{- Column, DataFusionError, Result, ScalarValue, TableReference,+ Column, DFSchema, DataFusionError, Result, ScalarValue, TableReference,assert_eq_or_internal_err, internal_err,tree_node::{Transformed, TransformedResult, TreeNode},};@@ -389,11 +389,16 @@ pub(crate) fn try_transform_to_simple_table_scan_with_filters(}LogicalPlan::TableScan(table_scan) => {let table_schema = table_scan.source.schema();+ let filter_schema = DFSchema::try_from_qualified_schema(+ table_scan.table_name.clone(),+ table_schema.as_ref(),+ )?;// optional rewriter if table has an aliaslet mut filter_alias_rewriter =table_alias.as_ref().map(|alias_name| TableAliasRewriter {- table_schema: &table_schema,+ table_schema: &filter_schema,alias_name: alias_name.clone(),+ rewrite_unqualified: true,});// rewrite filters to use table alias if presentdiff --git a/datafusion/sql/tests/cases/plan_to_sql.rs b/datafusion/sql/tests/cases/plan_to_sql.rsindex 03d12de046..937ed9894f 100644--- a/datafusion/sql/tests/cases/plan_to_sql.rs+++ b/datafusion/sql/tests/cases/plan_to_sql.rs@@ -2736,6 +2736,63 @@ fn test_unparse_inner_join_with_table_scan_projection() -> Result<()> {Ok(())}+#[test]+fn test_unparse_projected_join_unwraps_right_nested_passthrough_projection() -> Result<()>+{+ let left_schema = Schema::new(vec![+ Field::new("left_id", DataType::Int32, false),+ Field::new("mid_id", DataType::Int32, false),+ ]);+ let mid_schema = Schema::new(vec![+ Field::new("mid_id", DataType::Int32, false),+ Field::new("right_id", DataType::Int32, false),+ ]);+ let right_schema = Schema::new(vec![+ Field::new("right_id", DataType::Int32, false),+ Field::new("value", DataType::Int32, false),+ ]);++ let left = table_scan(Some("left_table"), &left_schema, None)?.build()?;+ let mid = table_scan(Some("mid_table"), &mid_schema, None)?.build()?;+ let right = table_scan(Some("right_table"), &right_schema, None)?.build()?;++ let nested_right = LogicalPlanBuilder::from(mid)+ .join(+ right,+ datafusion_expr::JoinType::Inner,+ (vec!["mid_table.right_id"], vec!["right_table.right_id"]),+ None,+ )?+ .project(vec![+ col("mid_table.mid_id"),+ col("mid_table.right_id"),+ col("right_table.value"),+ ])?+ .build()?;++ let plan = LogicalPlanBuilder::from(left)+ .join(+ nested_right,+ datafusion_expr::JoinType::Inner,+ (vec!["left_table.mid_id"], vec!["mid_table.mid_id"]),+ None,+ )?+ .project(vec![+ col("left_table.left_id"),+ col("mid_table.mid_id"),+ col("right_table.value"),+ ])?+ .build()?;++ let sql = plan_to_sql(&plan)?;+ assert_snapshot!(+ sql,+ @r#"SELECT left_table.left_id, mid_table.mid_id, right_table."value" FROM left_table INNER JOIN (mid_table INNER JOIN right_table ON mid_table.right_id = right_table.right_id) ON left_table.mid_id = mid_table.mid_id"#+ );++ Ok(())+}+#[test]fn test_unparse_left_semi_join_with_table_scan_projection() -> Result<()> {let schema = Schema::new(vec![