Skip to content

Apache DataFusion SQL Query Engine

Fix DuckDB unparse for optimized join projections (#23002)

Goutam Adwant48a5e21263d0
diff --git a/datafusion/core/tests/sql/unparser.rs b/datafusion/core/tests/sql/unparser.rs
index d6ca872e19..7762512fac 100644
--- a/datafusion/core/tests/sql/unparser.rs
+++ b/datafusion/core/tests/sql/unparser.rs
@@ -37,14 +37,19 @@
use std::fs::ReadDir;
use std::future::Future;
+use std::sync::Arc;
use arrow::array::RecordBatch;
+use arrow::datatypes::{DataType, Field, Schema};
use datafusion::common::Result;
+use datafusion::datasource::empty::EmptyTable;
use datafusion::prelude::{ParquetReadOptions, SessionContext};
+use datafusion_catalog::memory::MemorySchemaProvider;
+use datafusion_catalog::{CatalogProvider, MemoryCatalogProvider, SchemaProvider};
use datafusion_common::Column;
use datafusion_expr::Expr;
use datafusion_sql::unparser::Unparser;
-use datafusion_sql::unparser::dialect::DefaultDialect;
+use datafusion_sql::unparser::dialect::{DefaultDialect, DuckDBDialect};
use itertools::Itertools;
use recursive::{set_minimum_stack_size, set_stack_allocation_size};
@@ -218,6 +223,126 @@ async fn sort_batches(
df.collect().await
}
+const ISSUE_22961_QUERY: &str = r#"
+SELECT * FROM
+(
+SELECT
+ item_id,
+ order_id,
+ product_id,
+ quantity,
+ unit_price,
+ quantity * unit_price AS line_total
+ FROM
+ "warehouse"."main"."order_items"
+) oi
+JOIN (
+ SELECT
+ order_id,
+ customer_id,
+ order_date,
+ lower(STATUS) AS STATUS,
+ lower(channel) AS channel,
+ coalesce(discount_pct, 0) AS discount_pct,
+ coalesce(shipping_cost, 0) AS shipping_cost,
+ STATUS IN ('completed', 'shipped') AS is_fulfilled
+ FROM
+ "warehouse"."main"."orders"
+) o USING (order_id)
+JOIN (
+ SELECT
+ p.product_id,
+ p.category_id,
+ p.sku,
+ p.name AS product_name,
+ p.price,
+ p.cost,
+ p.weight_kg,
+ p.is_active,
+ p.stock_qty,
+ round(p.price - p.cost, 2) AS gross_margin,
+ round((p.price - p.cost) / nullif(p.price, 0), 4) AS margin_pct,
+ c.name AS category_name
+ FROM
+ "warehouse"."main"."products" p
+ LEFT JOIN "warehouse"."main"."categories" c USING (category_id)
+) p USING (product_id)
+"#;
+
+fn issue_22961_context() -> Result<SessionContext> {
+ let ctx = SessionContext::new();
+
+ let schema_provider = Arc::new(MemorySchemaProvider::new());
+ schema_provider.register_table(
+ "order_items".to_string(),
+ Arc::new(EmptyTable::new(Arc::new(Schema::new(vec![
+ Field::new("item_id", DataType::Int32, false),
+ Field::new("order_id", DataType::Int32, true),
+ Field::new("product_id", DataType::Int32, true),
+ Field::new("quantity", DataType::Int32, true),
+ Field::new("unit_price", DataType::Decimal128(10, 2), true),
+ ])))),
+ )?;
+ schema_provider.register_table(
+ "orders".to_string(),
+ Arc::new(EmptyTable::new(Arc::new(Schema::new(vec![
+ Field::new("order_id", DataType::Int32, false),
+ Field::new("customer_id", DataType::Int32, true),
+ Field::new("order_date", DataType::Date32, true),
+ Field::new("status", DataType::Utf8, true),
+ Field::new("channel", DataType::Utf8, true),
+ Field::new("discount_pct", DataType::Decimal128(5, 2), true),
+ Field::new("shipping_cost", DataType::Decimal128(8, 2), true),
+ ])))),
+ )?;
+ schema_provider.register_table(
+ "products".to_string(),
+ Arc::new(EmptyTable::new(Arc::new(Schema::new(vec![
+ Field::new("product_id", DataType::Int32, false),
+ Field::new("category_id", DataType::Int32, true),
+ Field::new("sku", DataType::Utf8, true),
+ Field::new("name", DataType::Utf8, true),
+ Field::new("price", DataType::Decimal128(10, 2), true),
+ Field::new("cost", DataType::Decimal128(10, 2), true),
+ Field::new("weight_kg", DataType::Decimal128(6, 3), true),
+ Field::new("is_active", DataType::Boolean, true),
+ Field::new("stock_qty", DataType::Int32, true),
+ ])))),
+ )?;
+ schema_provider.register_table(
+ "categories".to_string(),
+ Arc::new(EmptyTable::new(Arc::new(Schema::new(vec![
+ Field::new("category_id", DataType::Int32, false),
+ Field::new("name", DataType::Utf8, true),
+ Field::new("parent_id", DataType::Int32, true),
+ Field::new("display_rank", DataType::Int32, true),
+ ])))),
+ )?;
+
+ let catalog = Arc::new(MemoryCatalogProvider::new());
+ catalog.register_schema("main", schema_provider)?;
+ ctx.register_catalog("warehouse", catalog);
+
+ Ok(ctx)
+}
+
+#[tokio::test]
+async fn optimized_duckdb_unparse_preserves_derived_table_scope() -> Result<()> {
+ let ctx = issue_22961_context()?;
+ let plan = ctx.sql(ISSUE_22961_QUERY).await?.into_optimized_plan()?;
+ let dialect = DuckDBDialect::new();
+ let unparser = Unparser::new(&dialect);
+ let sql = unparser.plan_to_sql(&plan)?.to_string();
+
+ assert!(!sql.contains(r#""o"."__common_expr_1""#));
+ assert!(!sql.contains(r#""o"."__common_expr_2""#));
+ assert!(sql.contains(
+ r#"ON "oi"."order_id" = "o"."order_id" INNER JOIN (SELECT "p"."product_id""#
+ ));
+
+ Ok(())
+}
+
/// The outcome of running a single roundtrip test.
///
/// A successful test produces [`TestCaseResult::Success`].
diff --git a/datafusion/sql/src/unparser/ast.rs b/datafusion/sql/src/unparser/ast.rs
index 4b4e56c40c..7418d0b5b7 100644
--- a/datafusion/sql/src/unparser/ast.rs
+++ b/datafusion/sql/src/unparser/ast.rs
@@ -264,6 +264,9 @@ impl SelectBuilder {
pub fn pop_from(&mut self) -> Option<TableWithJoinsBuilder> {
self.from.pop()
}
+ pub fn has_selection(&self) -> bool {
+ self.selection.is_some()
+ }
pub fn lateral_views(&mut self, value: Vec<ast::LateralView>) -> &mut Self {
self.lateral_views = value;
self
@@ -483,6 +486,7 @@ pub struct RelationBuilder {
enum TableFactorBuilder {
Table(TableRelationBuilder),
Derived(DerivedRelationBuilder),
+ NestedJoin(ast::TableWithJoins, Option<ast::TableAlias>),
Unnest(UnnestRelationBuilder),
Flatten(FlattenRelationBuilder),
Empty,
@@ -501,6 +505,15 @@ impl RelationBuilder {
self
}
+ pub fn nested_join(
+ &mut self,
+ value: ast::TableWithJoins,
+ alias: Option<ast::TableAlias>,
+ ) -> &mut Self {
+ self.relation = Some(TableFactorBuilder::NestedJoin(value, alias));
+ self
+ }
+
pub fn unnest(&mut self, value: UnnestRelationBuilder) -> &mut Self {
self.relation = Some(TableFactorBuilder::Unnest(value));
self
@@ -524,6 +537,9 @@ impl RelationBuilder {
Some(TableFactorBuilder::Derived(ref mut rel_builder)) => {
rel_builder.alias = value;
}
+ Some(TableFactorBuilder::NestedJoin(_, ref mut alias)) => {
+ *alias = value;
+ }
Some(TableFactorBuilder::Unnest(ref mut rel_builder)) => {
rel_builder.alias = value;
}
@@ -539,6 +555,12 @@ impl RelationBuilder {
Ok(match self.relation {
Some(TableFactorBuilder::Table(ref value)) => Some(value.build()?),
Some(TableFactorBuilder::Derived(ref value)) => Some(value.build()?),
+ Some(TableFactorBuilder::NestedJoin(ref table_with_joins, ref alias)) => {
+ Some(ast::TableFactor::NestedJoin {
+ table_with_joins: Box::new(table_with_joins.clone()),
+ alias: alias.clone(),
+ })
+ }
Some(TableFactorBuilder::Unnest(ref value)) => Some(value.build()?),
Some(TableFactorBuilder::Flatten(ref value)) => Some(value.build()?),
Some(TableFactorBuilder::Empty) => None,
diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs
index 861de01e75..37ff8145d3 100644
--- a/datafusion/sql/src/unparser/plan.rs
+++ b/datafusion/sql/src/unparser/plan.rs
@@ -42,8 +42,8 @@ use crate::unparser::{
};
use crate::utils::UNNEST_PLACEHOLDER;
use datafusion_common::{
- Column, DataFusionError, Result, ScalarValue, TableReference, assert_or_internal_err,
- internal_datafusion_err, internal_err, not_impl_err,
+ Column, DFSchema, DataFusionError, Result, ScalarValue, TableReference,
+ assert_or_internal_err, internal_datafusion_err, internal_err, not_impl_err,
tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion},
utils::combine_limit,
};
@@ -571,8 +571,9 @@ impl Unparser<'_> {
let input_schema = window.input.schema();
let mut alias_rewriter = TableAliasRewriter {
- table_schema: input_schema.as_arrow(),
+ table_schema: input_schema.as_ref(),
alias_name: TableReference::bare(input_alias),
+ rewrite_unqualified: true,
};
let window_expr = window
.window_expr
@@ -1160,6 +1161,11 @@ impl Unparser<'_> {
}
None => Arc::clone(left_plan),
};
+ let left_plan = if already_projected {
+ Self::unwrap_qualified_passthrough_join_projection(left_plan)
+ } else {
+ left_plan
+ };
self.select_to_sql_recursively(
left_plan.as_ref(),
@@ -1185,13 +1191,22 @@ impl Unparser<'_> {
};
let mut right_relation = RelationBuilder::default();
-
- self.select_to_sql_recursively(
- right_plan.as_ref(),
- query,
- select,
- &mut right_relation,
- )?;
+ if already_projected
+ && let Some(nested_relation) = self
+ .qualified_passthrough_join_projection_to_nested_relation(
+ right_plan.as_ref(),
+ query,
+ )?
+ {
+ right_relation = nested_relation;
+ } else {
+ self.select_to_sql_recursively(
+ right_plan.as_ref(),
+ query,
+ select,
+ &mut right_relation,
+ )?;
+ }
let (join_filters, where_filters) = Self::split_join_on_and_where_filters(
join.join_type,
@@ -1910,6 +1925,68 @@ impl Unparser<'_> {
)
}
+ fn is_qualified_passthrough_projection(projection: &Projection) -> bool {
+ projection
+ .expr
+ .iter()
+ .all(|expr| matches!(expr, Expr::Column(column) if column.relation.is_some()))
+ }
+
+ fn unwrap_qualified_passthrough_join_projection(
+ plan: Arc<LogicalPlan>,
+ ) -> Arc<LogicalPlan> {
+ if let LogicalPlan::Projection(projection) = plan.as_ref()
+ && matches!(projection.input.as_ref(), LogicalPlan::Join(_))
+ && Self::is_qualified_passthrough_projection(projection)
+ {
+ Arc::clone(&projection.input)
+ } else {
+ plan
+ }
+ }
+
+ fn qualified_passthrough_join_projection_to_nested_relation(
+ &self,
+ plan: &LogicalPlan,
+ query: &mut Option<QueryBuilder>,
+ ) -> Result<Option<RelationBuilder>> {
+ let LogicalPlan::Projection(projection) = plan else {
+ return Ok(None);
+ };
+ if !matches!(projection.input.as_ref(), LogicalPlan::Join(_))
+ || !Self::is_qualified_passthrough_projection(projection)
+ {
+ return Ok(None);
+ }
+
+ let original_query = query.clone();
+ let mut nested_select = SelectBuilder::default();
+ nested_select.push_from(TableWithJoinsBuilder::default());
+ let mut nested_relation = RelationBuilder::default();
+ self.select_to_sql_recursively(
+ projection.input.as_ref(),
+ query,
+ &mut nested_select,
+ &mut nested_relation,
+ )?;
+ if nested_select.has_selection() {
+ *query = original_query;
+ return Ok(None);
+ }
+
+ let Some(mut nested_from) = nested_select.pop_from() else {
+ return internal_err!("Failed to build nested join relation");
+ };
+ nested_from.relation(nested_relation);
+ let Some(table_with_joins) = nested_from.build()? else {
+ return internal_err!("Failed to build nested join relation");
+ };
+
+ let mut relation = RelationBuilder::default();
+ relation.nested_join(table_with_joins, None);
+ Ok(Some(relation))
+ }
+
/// Try to unparse a table scan with pushdown operations into a new subquery plan.
/// If the table scan is without any pushdown operations, return None.
fn unparse_table_scan_pushdown(
@@ -1924,10 +2001,15 @@ impl Unparser<'_> {
return Ok(None);
}
let table_schema = table_scan.source.schema();
+ let filter_schema = DFSchema::try_from_qualified_schema(
+ table_scan.table_name.clone(),
+ table_schema.as_ref(),
+ )?;
let mut filter_alias_rewriter =
alias.as_ref().map(|alias_name| TableAliasRewriter {
- table_schema: &table_schema,
+ table_schema: &filter_schema,
alias_name: alias_name.clone(),
+ rewrite_unqualified: true,
});
let mut builder = LogicalPlanBuilder::scan(
@@ -2037,8 +2119,9 @@ impl Unparser<'_> {
let exprs = if alias.is_some() {
let mut alias_rewriter =
alias.as_ref().map(|alias_name| TableAliasRewriter {
- table_schema: plan.schema().as_arrow(),
+ table_schema: plan.schema().as_ref(),
alias_name: alias_name.clone(),
+ rewrite_unqualified: false,
});
projection
.expr
diff --git a/datafusion/sql/src/unparser/rewrite.rs b/datafusion/sql/src/unparser/rewrite.rs
index a6bfba4cca..6ee66f6193 100644
--- a/datafusion/sql/src/unparser/rewrite.rs
+++ b/datafusion/sql/src/unparser/rewrite.rs
@@ -17,10 +17,9 @@
use std::{collections::HashSet, sync::Arc};
-use arrow::datatypes::Schema;
use datafusion_common::tree_node::TreeNodeContainer;
use datafusion_common::{
- Column, HashMap, Result, TableReference,
+ Column, DFSchema, HashMap, Result, TableReference,
tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRewriter},
};
use datafusion_expr::expr::{Alias, UNNEST_COLUMN_PREFIX};
@@ -502,20 +501,24 @@ fn find_projection(logical_plan: &LogicalPlan) -> Option<&Projection> {
}
/// A `TreeNodeRewriter` implementation that rewrites `Expr::Column` expressions by
-/// replacing the column's name with an alias if the column exists in the provided schema.
+/// replacing the column's qualifier with an alias if the column resolves to a
+/// qualified field in the provided schema.
///
/// This is typically used to apply table aliases in query plans, ensuring that
/// the column references in the expressions use the correct table alias.
///
/// # Fields
///
-/// * `table_schema`: The schema (`SchemaRef`) representing the table structure
-/// from which the columns are referenced. This is used to look up columns by their names.
+/// * `table_schema`: The schema representing the table structure from which the
+/// columns are referenced. This is used to look up columns by their names and qualifiers.
/// * `alias_name`: The alias (`TableReference`) that will replace the table name
/// in the column references when applicable.
+/// * `rewrite_unqualified`: Whether columns that resolve to unqualified fields
+/// in `table_schema` should also be rewritten to `alias_name`.
pub struct TableAliasRewriter<'a> {
- pub table_schema: &'a Schema,
+ pub table_schema: &'a DFSchema,
pub alias_name: TableReference,
+ pub rewrite_unqualified: bool,
}
impl TreeNodeRewriter for TableAliasRewriter<'_> {
@@ -524,12 +527,23 @@ impl TreeNodeRewriter for TableAliasRewriter<'_> {
fn f_down(&mut self, expr: Expr) -> Result<Transformed<Expr>> {
match expr {
Expr::Column(column) => {
- if let Ok(field) = self.table_schema.field_with_name(&column.name) {
- let new_column =
- Column::new(Some(self.alias_name.clone()), field.name().clone());
- Ok(Transformed::yes(Expr::Column(new_column)))
- } else {
- Ok(Transformed::no(Expr::Column(column)))
+ match self
+ .table_schema
+ .qualified_field_from_column(&column)
+ .or_else(|_| {
+ self.table_schema
+ .qualified_field_with_unqualified_name(&column.name)
+ }) {
+ Ok((qualifier, field))
+ if qualifier.is_some() || self.rewrite_unqualified =>
+ {
+ let new_column = Column::new(
+ Some(self.alias_name.clone()),
+ field.name().clone(),
+ );
+ Ok(Transformed::yes(Expr::Column(new_column)))
+ }
+ Ok(_) | Err(_) => Ok(Transformed::no(Expr::Column(column))),
}
}
_ => Ok(Transformed::no(expr)),
@@ -540,7 +554,7 @@ impl TreeNodeRewriter for TableAliasRewriter<'_> {
#[cfg(test)]
mod tests {
use super::*;
- use arrow::datatypes::{DataType, Field};
+ use arrow::datatypes::{DataType, Field, Schema};
use datafusion_expr::{LogicalPlanBuilder, col, table_scan};
// this is a regression test: when the outer projection has fewer expressions than
diff --git a/datafusion/sql/src/unparser/utils.rs b/datafusion/sql/src/unparser/utils.rs
index 1cc023d112..949b49eb77 100644
--- a/datafusion/sql/src/unparser/utils.rs
+++ b/datafusion/sql/src/unparser/utils.rs
@@ -22,7 +22,7 @@ use super::{
rewrite::TableAliasRewriter,
};
use datafusion_common::{
- Column, DataFusionError, Result, ScalarValue, TableReference,
+ Column, DFSchema, DataFusionError, Result, ScalarValue, TableReference,
assert_eq_or_internal_err, internal_err,
tree_node::{Transformed, TransformedResult, TreeNode},
};
@@ -389,11 +389,16 @@ pub(crate) fn try_transform_to_simple_table_scan_with_filters(
}
LogicalPlan::TableScan(table_scan) => {
let table_schema = table_scan.source.schema();
+ let filter_schema = DFSchema::try_from_qualified_schema(
+ table_scan.table_name.clone(),
+ table_schema.as_ref(),
+ )?;
// optional rewriter if table has an alias
let mut filter_alias_rewriter =
table_alias.as_ref().map(|alias_name| TableAliasRewriter {
- table_schema: &table_schema,
+ table_schema: &filter_schema,
alias_name: alias_name.clone(),
+ rewrite_unqualified: true,
});
// rewrite filters to use table alias if present
diff --git a/datafusion/sql/tests/cases/plan_to_sql.rs b/datafusion/sql/tests/cases/plan_to_sql.rs
index 03d12de046..937ed9894f 100644
--- a/datafusion/sql/tests/cases/plan_to_sql.rs
+++ b/datafusion/sql/tests/cases/plan_to_sql.rs
@@ -2736,6 +2736,63 @@ fn test_unparse_inner_join_with_table_scan_projection() -> Result<()> {
Ok(())
}
+#[test]
+fn test_unparse_projected_join_unwraps_right_nested_passthrough_projection() -> Result<()>
+{
+ let left_schema = Schema::new(vec![
+ Field::new("left_id", DataType::Int32, false),
+ Field::new("mid_id", DataType::Int32, false),
+ ]);
+ let mid_schema = Schema::new(vec![
+ Field::new("mid_id", DataType::Int32, false),
+ Field::new("right_id", DataType::Int32, false),
+ ]);
+ let right_schema = Schema::new(vec![
+ Field::new("right_id", DataType::Int32, false),
+ Field::new("value", DataType::Int32, false),
+ ]);
+
+ let left = table_scan(Some("left_table"), &left_schema, None)?.build()?;
+ let mid = table_scan(Some("mid_table"), &mid_schema, None)?.build()?;
+ let right = table_scan(Some("right_table"), &right_schema, None)?.build()?;
+
+ let nested_right = LogicalPlanBuilder::from(mid)
+ .join(
+ right,
+ datafusion_expr::JoinType::Inner,
+ (vec!["mid_table.right_id"], vec!["right_table.right_id"]),
+ None,
+ )?
+ .project(vec![
+ col("mid_table.mid_id"),
+ col("mid_table.right_id"),
+ col("right_table.value"),
+ ])?
+ .build()?;
+
+ let plan = LogicalPlanBuilder::from(left)
+ .join(
+ nested_right,
+ datafusion_expr::JoinType::Inner,
+ (vec!["left_table.mid_id"], vec!["mid_table.mid_id"]),
+ None,
+ )?
+ .project(vec![
+ col("left_table.left_id"),
+ col("mid_table.mid_id"),
+ col("right_table.value"),
+ ])?
+ .build()?;
+
+ let sql = plan_to_sql(&plan)?;
+ assert_snapshot!(
+ sql,
+ @r#"SELECT left_table.left_id, mid_table.mid_id, right_table."value" FROM left_table INNER JOIN (mid_table INNER JOIN right_table ON mid_table.right_id = right_table.right_id) ON left_table.mid_id = mid_table.mid_id"#
+ );
+
+ Ok(())
+}
+
#[test]
fn test_unparse_left_semi_join_with_table_scan_projection() -> Result<()> {
let schema = Schema::new(vec![