Apache DataFusion SQL Query Engine
fix: preserve aggregate scope when unparsing (#23327)
Phoenixa29f70c5b704
diff --git a/datafusion/core/tests/sql/unparser.rs b/datafusion/core/tests/sql/unparser.rsindex 4597b7e640..d689fb1496 100644--- a/datafusion/core/tests/sql/unparser.rs+++ b/datafusion/core/tests/sql/unparser.rs@@ -43,6 +43,9 @@ use arrow::array::RecordBatch;use arrow::datatypes::{DataType, Field, Schema};use datafusion::common::Result;use datafusion::datasource::empty::EmptyTable;+use datafusion::optimizer::{+ OptimizerRule, single_distinct_to_groupby::SingleDistinctToGroupBy,+};use datafusion::prelude::{ParquetReadOptions, SessionContext};use datafusion_catalog::memory::MemorySchemaProvider;use datafusion_catalog::{CatalogProvider, MemoryCatalogProvider, SchemaProvider};@@ -399,6 +402,216 @@ async fn optimized_duckdb_unparse_qualifies_nested_passthrough_column() -> ResulOk(())}+// https://github.com/apache/datafusion/issues/23317+//+// `SingleDistinctToGroupBy` rewrites single DISTINCT aggregates into a+// two-phase aggregate plan. The inner Aggregate defines intermediate fields+// such as `group_alias_0`, `alias1`, and `alias2`. The unparser must preserve+// that inner Aggregate as a derived table before the outer Aggregate+// references those fields.+//+// Without `SingleDistinctToGroupBy`, the Aggregate still sits over an unnamed+// derived Projection. In that SQL scope, base table aliases `cs` and `c` are no+// longer visible, so aggregate expressions must refer to the derived table's+// output columns unqualified.+const ISSUE_23317_QUERY: &str = r#"+WITH cohort AS (+ SELECT+ signup_year,+ sum(customers) AS customers,+ sum(revenue) AS revenue+ FROM+ (+ SELECT+ date_part('year', c.signup_date) AS signup_year,+ count(DISTINCT cs.customer_id) AS customers,+ round(sum(cs.total_revenue), 2) AS revenue+ FROM+ "warehouse"."main"."sales" cs+ JOIN "warehouse"."main"."customers" c USING (customer_id)+ GROUP BY+ 1+ )+ GROUP BY+ signup_year+)+SELECT+ *+FROM+ cohort+"#;++const ISSUE_23317_HAVING_QUERY: &str = r#"+SELECT+ date_part('year', c.signup_date) AS signup_year,+ count(DISTINCT cs.customer_id) AS customers+FROM+ "warehouse"."main"."sales" cs+ JOIN "warehouse"."main"."customers" c USING (customer_id)+GROUP BY+ 1+HAVING+ count(DISTINCT cs.customer_id) > 0+"#;++const ISSUE_23317_QUALIFY_QUERY: &str = r#"+SELECT+ date_part('year', c.signup_date) AS signup_year,+ count(DISTINCT cs.customer_id) AS customers,+ row_number() OVER (ORDER BY date_part('year', c.signup_date)) AS rn+FROM+ "warehouse"."main"."sales" cs+ JOIN "warehouse"."main"."customers" c USING (customer_id)+GROUP BY+ 1+QUALIFY+ rn = 1 AND count(DISTINCT cs.customer_id) > 0+"#;++fn issue_23317_context() -> Result<SessionContext> {+ let ctx = SessionContext::new();++ let schema_provider = Arc::new(MemorySchemaProvider::new());+ schema_provider.register_table(+ "customers".to_string(),+ Arc::new(EmptyTable::new(Arc::new(Schema::new(vec![+ Field::new("customer_id", DataType::Int32, false),+ Field::new("signup_date", DataType::Date32, true),+ ])))),+ )?;+ schema_provider.register_table(+ "sales".to_string(),+ Arc::new(EmptyTable::new(Arc::new(Schema::new(vec![+ Field::new("customer_id", DataType::Int32, false),+ Field::new("total_revenue", DataType::Decimal128(12, 2), true),+ ])))),+ )?;++ let catalog = Arc::new(MemoryCatalogProvider::new());+ catalog.register_schema("main", schema_provider)?;+ ctx.register_catalog("warehouse", catalog);++ Ok(ctx)+}++async fn assert_issue_23317_unparsed_sql_plans(+ ctx: &SessionContext,+ sql: &str,+) -> Result<()> {+ ctx.sql(sql).await?.into_optimized_plan()?;+ Ok(())+}++#[tokio::test]+async fn optimized_duckdb_unparse_preserves_nested_aggregate_scope() -> Result<()> {+ let ctx = issue_23317_context()?;+ let plan = ctx.sql(ISSUE_23317_QUERY).await?.into_optimized_plan()?;+ let dialect = DuckDBDialect::new();+ let unparser = Unparser::new(&dialect);+ let sql = unparser.plan_to_sql(&plan)?.to_string();++ assert_issue_23317_unparsed_sql_plans(&ctx, &sql).await?;++ assert!(+ sql.contains(concat!(+ r#"FROM (SELECT sum("total_revenue") AS "alias2", "#,+ r#"date_part('year', "signup_date") AS "group_alias_0", "#,+ r#""customer_id" AS "alias1" "#+ )),+ "inner aggregate should define the aliases before the outer aggregate uses them: {sql}",+ );+ assert!(+ !sql.contains(r#"date_part('year', "c"."signup_date") AS "group_alias_0""#),+ "inner aggregate must not reference out-of-scope alias c: {sql}",+ );++ Ok(())+}++#[tokio::test]+async fn optimized_duckdb_unparse_unqualifies_aggregate_input_projection() -> Result<()> {+ let ctx = issue_23317_context()?;+ assert!(ctx.remove_optimizer_rule(SingleDistinctToGroupBy::new().name()));++ let plan = ctx.sql(ISSUE_23317_QUERY).await?.into_optimized_plan()?;+ let dialect = DuckDBDialect::new();+ let unparser = Unparser::new(&dialect);+ let sql = unparser.plan_to_sql(&plan)?.to_string();++ assert_issue_23317_unparsed_sql_plans(&ctx, &sql).await?;++ assert!(+ sql.contains(+ r#"SELECT date_part('year', "signup_date") AS "signup_year", count(DISTINCT "customer_id") AS "customers", round(sum("total_revenue"), 2) AS "revenue" FROM ("#+ ),+ "aggregate expressions should resolve against the derived projection output: {sql}",+ );+ assert!(+ !sql.contains(r#"date_part('year', "c"."signup_date") AS "signup_year""#),+ "derived aggregate must not reference out-of-scope alias c: {sql}",+ );++ Ok(())+}++#[tokio::test]+async fn optimized_duckdb_unparse_having_unqualifies_agg_input() -> Result<()> {+ let ctx = issue_23317_context()?;+ assert!(ctx.remove_optimizer_rule(SingleDistinctToGroupBy::new().name()));++ let plan = ctx+ .sql(ISSUE_23317_HAVING_QUERY)+ .await?+ .into_optimized_plan()?;+ let dialect = DuckDBDialect::new();+ let unparser = Unparser::new(&dialect);+ let sql = unparser.plan_to_sql(&plan)?.to_string();++ assert_issue_23317_unparsed_sql_plans(&ctx, &sql).await?;++ assert!(+ sql.contains(r#"HAVING (count(DISTINCT "customer_id") > 0)"#),+ "HAVING aggregate should resolve against the derived projection output: {sql}",+ );+ assert!(+ !sql.contains(r#"count(DISTINCT "cs"."customer_id")"#),+ "HAVING must not reference out-of-scope alias cs: {sql}",+ );++ Ok(())+}++#[tokio::test]+async fn optimized_duckdb_unparse_qualify_unqualifies_agg_input() -> Result<()> {+ let ctx = issue_23317_context()?;+ assert!(ctx.remove_optimizer_rule(SingleDistinctToGroupBy::new().name()));++ let plan = ctx+ .sql(ISSUE_23317_QUALIFY_QUERY)+ .await?+ .into_optimized_plan()?;+ let dialect = DuckDBDialect::new();+ let unparser = Unparser::new(&dialect);+ let sql = unparser.plan_to_sql(&plan)?.to_string();++ assert_issue_23317_unparsed_sql_plans(&ctx, &sql).await?;++ assert!(+ sql.contains("QUALIFY"),+ "expected QUALIFY clause in unparsed SQL: {sql}",+ );+ assert!(+ sql.contains(r#"count(DISTINCT "customer_id") > 0"#),+ "QUALIFY aggregate should resolve against the derived projection output: {sql}",+ );+ assert!(+ !sql.contains(r#"count(DISTINCT "cs"."customer_id")"#),+ "QUALIFY must not reference out-of-scope alias cs: {sql}",+ );++ Ok(())+}+/// The outcome of running a single roundtrip test.////// A successful test produces [`TestCaseResult::Success`].diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rsindex b538b31a76..5eef9b82d9 100644--- a/datafusion/sql/src/unparser/plan.rs+++ b/datafusion/sql/src/unparser/plan.rs@@ -312,10 +312,17 @@ impl Unparser<'_> {match (agg, window) {(Some(agg), window) => {let window_option = window.as_deref();+ let agg_input_has_derived_projection =+ Self::contains_projection_before_relation(agg.input.as_ref());let items = exprs.into_iter().map(|proj_expr| {let unproj = unproject_agg_exprs(proj_expr, agg, window_option)?;+ let unproj = Self::normalize_agg_input_columns(+ unproj,+ agg,+ agg_input_has_derived_projection,+ )?;self.select_item_to_sql(&unproj)}).collect::<Result<Vec<_>>>()?;@@ -324,7 +331,15 @@ impl Unparser<'_> {select.group_by(ast::GroupByExpr::Expressions(agg.group_expr.iter()- .map(|expr| self.expr_to_sql(expr))+ .cloned()+ .map(|expr| {+ let expr = Self::normalize_agg_input_columns(+ expr,+ agg,+ agg_input_has_derived_projection,+ )?;+ self.expr_to_sql(&expr)+ }).collect::<Result<Vec<_>>>()?,vec![],));@@ -364,6 +379,56 @@ impl Unparser<'_> {}}+ fn normalize_agg_input_columns(+ expr: Expr,+ agg: &Aggregate,+ input_has_derived_projection: bool,+ ) -> Result<Expr> {+ if input_has_derived_projection {+ Self::strip_column_qualifiers_for_schema(expr, agg.input.schema().as_ref())+ } else {+ Ok(expr)+ }+ }++ fn contains_projection_before_relation(plan: &LogicalPlan) -> bool {+ match plan {+ LogicalPlan::Projection(_) => true,+ LogicalPlan::TableScan(_)+ | LogicalPlan::Subquery(_)+ | LogicalPlan::SubqueryAlias(_)+ | LogicalPlan::Join(_)+ | LogicalPlan::EmptyRelation(_)+ | LogicalPlan::Values(_) => false,+ _ => {+ let inputs = plan.inputs();+ matches!(+ inputs.as_slice(),+ [input] if Self::contains_projection_before_relation(input)+ )+ }+ }+ }++ fn contains_aggregate_before_relation(plan: &LogicalPlan) -> bool {+ match plan {+ LogicalPlan::Aggregate(_) => true,+ LogicalPlan::TableScan(_)+ | LogicalPlan::Subquery(_)+ | LogicalPlan::SubqueryAlias(_)+ | LogicalPlan::Join(_)+ | LogicalPlan::EmptyRelation(_)+ | LogicalPlan::Values(_) => false,+ _ => {+ let inputs = plan.inputs();+ matches!(+ inputs.as_slice(),+ [input] if Self::contains_aggregate_before_relation(input)+ )+ }+ }+ }+fn derive(&self,plan: &LogicalPlan,@@ -964,12 +1029,22 @@ impl Unparser<'_> {unproject_window_exprs(filter.predicate.clone(), window)?;if let Some(agg) = agg {unprojected = unproject_agg_exprs(unprojected, agg, None)?;+ unprojected = Self::normalize_agg_input_columns(+ unprojected,+ agg,+ Self::contains_projection_before_relation(agg.input.as_ref()),+ )?;}let filter_expr = self.expr_to_sql(&unprojected)?;select.qualify(Some(filter_expr));} else if let Some(agg) = agg {let unprojected =unproject_agg_exprs(filter.predicate.clone(), agg, None)?;+ let unprojected = Self::normalize_agg_input_columns(+ unprojected,+ agg,+ Self::contains_projection_before_relation(agg.input.as_ref()),+ )?;let filter_expr = self.expr_to_sql(&unprojected)?;select.having(Some(filter_expr));} else {@@ -1071,23 +1146,49 @@ impl Unparser<'_> {LogicalPlan::Aggregate(agg) => {// Aggregation can be already handled in the projection caseif !select.already_projected() {+ let agg_input_has_derived_projection =+ Self::contains_projection_before_relation(agg.input.as_ref());// The query returns aggregate and group expressions. If that weren't the case,// the aggregate would have been placed inside a projection, making the check above^ falselet exprs: Vec<_> = agg.aggr_expr.iter().chain(agg.group_expr.iter())- .map(|expr| self.select_item_to_sql(expr))+ .cloned()+ .map(|expr| {+ let expr = Self::normalize_agg_input_columns(+ expr,+ agg,+ agg_input_has_derived_projection,+ )?;+ self.select_item_to_sql(&expr)+ }).collect::<Result<Vec<_>>>()?;select.projection(exprs);select.group_by(ast::GroupByExpr::Expressions(agg.group_expr.iter()- .map(|expr| self.expr_to_sql(expr))+ .cloned()+ .map(|expr| {+ let expr = Self::normalize_agg_input_columns(+ expr,+ agg,+ agg_input_has_derived_projection,+ )?;+ self.expr_to_sql(&expr)+ }).collect::<Result<Vec<_>>>()?,vec![],));+ } else if Self::contains_aggregate_before_relation(agg.input.as_ref()) {+ return self.derive_with_dialect_alias(+ "derived_aggregate",+ agg.input.as_ref(),+ relation,+ false,+ vec![],+ );}self.select_to_sql_recursively(@@ -1992,11 +2093,10 @@ impl Unparser<'_> {Ok(Some(relation))}- /// Strip the table qualifier from every column in a pushdown pass-through- /// projection expression, so it resolves against the unnamed derived table- /// rendered for the inner pushdown projection rather than a deeper table- /// alias that is out of scope at this nesting level.- fn strip_pushdown_column_qualifiers(expr: Expr) -> Result<Expr> {+ /// Strip the table qualifier from every column in an expression that must+ /// resolve against an unnamed derived table's output columns rather than a+ /// deeper table alias that is out of scope at this nesting level.+ fn strip_column_qualifiers(expr: Expr) -> Result<Expr> {expr.transform(|e| match e {Expr::Column(mut column) => {column.relation = None;@@ -2007,6 +2107,20 @@ impl Unparser<'_> {.data()}+ fn strip_column_qualifiers_for_schema(expr: Expr, schema: &DFSchema) -> Result<Expr> {+ expr.transform(|e| match e {+ Expr::Column(mut column)+ if column.relation.is_some()+ && schema.index_of_column(&column).is_ok() =>+ {+ column.relation = None;+ Ok(Transformed::yes(Expr::Column(column)))+ }+ other => Ok(Transformed::no(other)),+ })+ .data()+ }+/// Try to unparse a table scan with pushdown operations into a new subquery plan./// If the table scan is without any pushdown operations, return None.fn unparse_table_scan_pushdown(@@ -2155,7 +2269,7 @@ impl Unparser<'_> {.expr.iter().cloned()- .map(Self::strip_pushdown_column_qualifiers)+ .map(Self::strip_column_qualifiers).collect::<Result<Vec<_>>>()?;return Ok(Some(LogicalPlan::Projection(Projection::try_new(exprs,