Skip to content

Apache DataFusion SQL Query Engine

fix: preserve aggregate scope when unparsing (#23327)

Phoenixa29f70c5b704
diff --git a/datafusion/core/tests/sql/unparser.rs b/datafusion/core/tests/sql/unparser.rs
index 4597b7e640..d689fb1496 100644
--- a/datafusion/core/tests/sql/unparser.rs
+++ b/datafusion/core/tests/sql/unparser.rs
@@ -43,6 +43,9 @@ use arrow::array::RecordBatch;
use arrow::datatypes::{DataType, Field, Schema};
use datafusion::common::Result;
use datafusion::datasource::empty::EmptyTable;
+use datafusion::optimizer::{
+ OptimizerRule, single_distinct_to_groupby::SingleDistinctToGroupBy,
+};
use datafusion::prelude::{ParquetReadOptions, SessionContext};
use datafusion_catalog::memory::MemorySchemaProvider;
use datafusion_catalog::{CatalogProvider, MemoryCatalogProvider, SchemaProvider};
@@ -399,6 +402,216 @@ async fn optimized_duckdb_unparse_qualifies_nested_passthrough_column() -> Resul
Ok(())
}
+// https://github.com/apache/datafusion/issues/23317
+//
+// `SingleDistinctToGroupBy` rewrites single DISTINCT aggregates into a
+// two-phase aggregate plan. The inner Aggregate defines intermediate fields
+// such as `group_alias_0`, `alias1`, and `alias2`. The unparser must preserve
+// that inner Aggregate as a derived table before the outer Aggregate
+// references those fields.
+//
+// Without `SingleDistinctToGroupBy`, the Aggregate still sits over an unnamed
+// derived Projection. In that SQL scope, base table aliases `cs` and `c` are no
+// longer visible, so aggregate expressions must refer to the derived table's
+// output columns unqualified.
+const ISSUE_23317_QUERY: &str = r#"
+WITH cohort AS (
+ SELECT
+ signup_year,
+ sum(customers) AS customers,
+ sum(revenue) AS revenue
+ FROM
+ (
+ SELECT
+ date_part('year', c.signup_date) AS signup_year,
+ count(DISTINCT cs.customer_id) AS customers,
+ round(sum(cs.total_revenue), 2) AS revenue
+ FROM
+ "warehouse"."main"."sales" cs
+ JOIN "warehouse"."main"."customers" c USING (customer_id)
+ GROUP BY
+ 1
+ )
+ GROUP BY
+ signup_year
+)
+SELECT
+ *
+FROM
+ cohort
+"#;
+
+const ISSUE_23317_HAVING_QUERY: &str = r#"
+SELECT
+ date_part('year', c.signup_date) AS signup_year,
+ count(DISTINCT cs.customer_id) AS customers
+FROM
+ "warehouse"."main"."sales" cs
+ JOIN "warehouse"."main"."customers" c USING (customer_id)
+GROUP BY
+ 1
+HAVING
+ count(DISTINCT cs.customer_id) > 0
+"#;
+
+const ISSUE_23317_QUALIFY_QUERY: &str = r#"
+SELECT
+ date_part('year', c.signup_date) AS signup_year,
+ count(DISTINCT cs.customer_id) AS customers,
+ row_number() OVER (ORDER BY date_part('year', c.signup_date)) AS rn
+FROM
+ "warehouse"."main"."sales" cs
+ JOIN "warehouse"."main"."customers" c USING (customer_id)
+GROUP BY
+ 1
+QUALIFY
+ rn = 1 AND count(DISTINCT cs.customer_id) > 0
+"#;
+
+fn issue_23317_context() -> Result<SessionContext> {
+ let ctx = SessionContext::new();
+
+ let schema_provider = Arc::new(MemorySchemaProvider::new());
+ schema_provider.register_table(
+ "customers".to_string(),
+ Arc::new(EmptyTable::new(Arc::new(Schema::new(vec![
+ Field::new("customer_id", DataType::Int32, false),
+ Field::new("signup_date", DataType::Date32, true),
+ ])))),
+ )?;
+ schema_provider.register_table(
+ "sales".to_string(),
+ Arc::new(EmptyTable::new(Arc::new(Schema::new(vec![
+ Field::new("customer_id", DataType::Int32, false),
+ Field::new("total_revenue", DataType::Decimal128(12, 2), true),
+ ])))),
+ )?;
+
+ let catalog = Arc::new(MemoryCatalogProvider::new());
+ catalog.register_schema("main", schema_provider)?;
+ ctx.register_catalog("warehouse", catalog);
+
+ Ok(ctx)
+}
+
+async fn assert_issue_23317_unparsed_sql_plans(
+ ctx: &SessionContext,
+ sql: &str,
+) -> Result<()> {
+ ctx.sql(sql).await?.into_optimized_plan()?;
+ Ok(())
+}
+
+#[tokio::test]
+async fn optimized_duckdb_unparse_preserves_nested_aggregate_scope() -> Result<()> {
+ let ctx = issue_23317_context()?;
+ let plan = ctx.sql(ISSUE_23317_QUERY).await?.into_optimized_plan()?;
+ let dialect = DuckDBDialect::new();
+ let unparser = Unparser::new(&dialect);
+ let sql = unparser.plan_to_sql(&plan)?.to_string();
+
+ assert_issue_23317_unparsed_sql_plans(&ctx, &sql).await?;
+
+ assert!(
+ sql.contains(concat!(
+ r#"FROM (SELECT sum("total_revenue") AS "alias2", "#,
+ r#"date_part('year', "signup_date") AS "group_alias_0", "#,
+ r#""customer_id" AS "alias1" "#
+ )),
+ "inner aggregate should define the aliases before the outer aggregate uses them: {sql}",
+ );
+ assert!(
+ !sql.contains(r#"date_part('year', "c"."signup_date") AS "group_alias_0""#),
+ "inner aggregate must not reference out-of-scope alias c: {sql}",
+ );
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn optimized_duckdb_unparse_unqualifies_aggregate_input_projection() -> Result<()> {
+ let ctx = issue_23317_context()?;
+ assert!(ctx.remove_optimizer_rule(SingleDistinctToGroupBy::new().name()));
+
+ let plan = ctx.sql(ISSUE_23317_QUERY).await?.into_optimized_plan()?;
+ let dialect = DuckDBDialect::new();
+ let unparser = Unparser::new(&dialect);
+ let sql = unparser.plan_to_sql(&plan)?.to_string();
+
+ assert_issue_23317_unparsed_sql_plans(&ctx, &sql).await?;
+
+ assert!(
+ sql.contains(
+ r#"SELECT date_part('year', "signup_date") AS "signup_year", count(DISTINCT "customer_id") AS "customers", round(sum("total_revenue"), 2) AS "revenue" FROM ("#
+ ),
+ "aggregate expressions should resolve against the derived projection output: {sql}",
+ );
+ assert!(
+ !sql.contains(r#"date_part('year', "c"."signup_date") AS "signup_year""#),
+ "derived aggregate must not reference out-of-scope alias c: {sql}",
+ );
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn optimized_duckdb_unparse_having_unqualifies_agg_input() -> Result<()> {
+ let ctx = issue_23317_context()?;
+ assert!(ctx.remove_optimizer_rule(SingleDistinctToGroupBy::new().name()));
+
+ let plan = ctx
+ .sql(ISSUE_23317_HAVING_QUERY)
+ .await?
+ .into_optimized_plan()?;
+ let dialect = DuckDBDialect::new();
+ let unparser = Unparser::new(&dialect);
+ let sql = unparser.plan_to_sql(&plan)?.to_string();
+
+ assert_issue_23317_unparsed_sql_plans(&ctx, &sql).await?;
+
+ assert!(
+ sql.contains(r#"HAVING (count(DISTINCT "customer_id") > 0)"#),
+ "HAVING aggregate should resolve against the derived projection output: {sql}",
+ );
+ assert!(
+ !sql.contains(r#"count(DISTINCT "cs"."customer_id")"#),
+ "HAVING must not reference out-of-scope alias cs: {sql}",
+ );
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn optimized_duckdb_unparse_qualify_unqualifies_agg_input() -> Result<()> {
+ let ctx = issue_23317_context()?;
+ assert!(ctx.remove_optimizer_rule(SingleDistinctToGroupBy::new().name()));
+
+ let plan = ctx
+ .sql(ISSUE_23317_QUALIFY_QUERY)
+ .await?
+ .into_optimized_plan()?;
+ let dialect = DuckDBDialect::new();
+ let unparser = Unparser::new(&dialect);
+ let sql = unparser.plan_to_sql(&plan)?.to_string();
+
+ assert_issue_23317_unparsed_sql_plans(&ctx, &sql).await?;
+
+ assert!(
+ sql.contains("QUALIFY"),
+ "expected QUALIFY clause in unparsed SQL: {sql}",
+ );
+ assert!(
+ sql.contains(r#"count(DISTINCT "customer_id") > 0"#),
+ "QUALIFY aggregate should resolve against the derived projection output: {sql}",
+ );
+ assert!(
+ !sql.contains(r#"count(DISTINCT "cs"."customer_id")"#),
+ "QUALIFY must not reference out-of-scope alias cs: {sql}",
+ );
+
+ Ok(())
+}
+
/// The outcome of running a single roundtrip test.
///
/// A successful test produces [`TestCaseResult::Success`].
diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs
index b538b31a76..5eef9b82d9 100644
--- a/datafusion/sql/src/unparser/plan.rs
+++ b/datafusion/sql/src/unparser/plan.rs
@@ -312,10 +312,17 @@ impl Unparser<'_> {
match (agg, window) {
(Some(agg), window) => {
let window_option = window.as_deref();
+ let agg_input_has_derived_projection =
+ Self::contains_projection_before_relation(agg.input.as_ref());
let items = exprs
.into_iter()
.map(|proj_expr| {
let unproj = unproject_agg_exprs(proj_expr, agg, window_option)?;
+ let unproj = Self::normalize_agg_input_columns(
+ unproj,
+ agg,
+ agg_input_has_derived_projection,
+ )?;
self.select_item_to_sql(&unproj)
})
.collect::<Result<Vec<_>>>()?;
@@ -324,7 +331,15 @@ impl Unparser<'_> {
select.group_by(ast::GroupByExpr::Expressions(
agg.group_expr
.iter()
- .map(|expr| self.expr_to_sql(expr))
+ .cloned()
+ .map(|expr| {
+ let expr = Self::normalize_agg_input_columns(
+ expr,
+ agg,
+ agg_input_has_derived_projection,
+ )?;
+ self.expr_to_sql(&expr)
+ })
.collect::<Result<Vec<_>>>()?,
vec![],
));
@@ -364,6 +379,56 @@ impl Unparser<'_> {
}
}
+ fn normalize_agg_input_columns(
+ expr: Expr,
+ agg: &Aggregate,
+ input_has_derived_projection: bool,
+ ) -> Result<Expr> {
+ if input_has_derived_projection {
+ Self::strip_column_qualifiers_for_schema(expr, agg.input.schema().as_ref())
+ } else {
+ Ok(expr)
+ }
+ }
+
+ fn contains_projection_before_relation(plan: &LogicalPlan) -> bool {
+ match plan {
+ LogicalPlan::Projection(_) => true,
+ LogicalPlan::TableScan(_)
+ | LogicalPlan::Subquery(_)
+ | LogicalPlan::SubqueryAlias(_)
+ | LogicalPlan::Join(_)
+ | LogicalPlan::EmptyRelation(_)
+ | LogicalPlan::Values(_) => false,
+ _ => {
+ let inputs = plan.inputs();
+ matches!(
+ inputs.as_slice(),
+ [input] if Self::contains_projection_before_relation(input)
+ )
+ }
+ }
+ }
+
+ fn contains_aggregate_before_relation(plan: &LogicalPlan) -> bool {
+ match plan {
+ LogicalPlan::Aggregate(_) => true,
+ LogicalPlan::TableScan(_)
+ | LogicalPlan::Subquery(_)
+ | LogicalPlan::SubqueryAlias(_)
+ | LogicalPlan::Join(_)
+ | LogicalPlan::EmptyRelation(_)
+ | LogicalPlan::Values(_) => false,
+ _ => {
+ let inputs = plan.inputs();
+ matches!(
+ inputs.as_slice(),
+ [input] if Self::contains_aggregate_before_relation(input)
+ )
+ }
+ }
+ }
+
fn derive(
&self,
plan: &LogicalPlan,
@@ -964,12 +1029,22 @@ impl Unparser<'_> {
unproject_window_exprs(filter.predicate.clone(), window)?;
if let Some(agg) = agg {
unprojected = unproject_agg_exprs(unprojected, agg, None)?;
+ unprojected = Self::normalize_agg_input_columns(
+ unprojected,
+ agg,
+ Self::contains_projection_before_relation(agg.input.as_ref()),
+ )?;
}
let filter_expr = self.expr_to_sql(&unprojected)?;
select.qualify(Some(filter_expr));
} else if let Some(agg) = agg {
let unprojected =
unproject_agg_exprs(filter.predicate.clone(), agg, None)?;
+ let unprojected = Self::normalize_agg_input_columns(
+ unprojected,
+ agg,
+ Self::contains_projection_before_relation(agg.input.as_ref()),
+ )?;
let filter_expr = self.expr_to_sql(&unprojected)?;
select.having(Some(filter_expr));
} else {
@@ -1071,23 +1146,49 @@ impl Unparser<'_> {
LogicalPlan::Aggregate(agg) => {
// Aggregation can be already handled in the projection case
if !select.already_projected() {
+ let agg_input_has_derived_projection =
+ Self::contains_projection_before_relation(agg.input.as_ref());
// The query returns aggregate and group expressions. If that weren't the case,
// the aggregate would have been placed inside a projection, making the check above^ false
let exprs: Vec<_> = agg
.aggr_expr
.iter()
.chain(agg.group_expr.iter())
- .map(|expr| self.select_item_to_sql(expr))
+ .cloned()
+ .map(|expr| {
+ let expr = Self::normalize_agg_input_columns(
+ expr,
+ agg,
+ agg_input_has_derived_projection,
+ )?;
+ self.select_item_to_sql(&expr)
+ })
.collect::<Result<Vec<_>>>()?;
select.projection(exprs);
select.group_by(ast::GroupByExpr::Expressions(
agg.group_expr
.iter()
- .map(|expr| self.expr_to_sql(expr))
+ .cloned()
+ .map(|expr| {
+ let expr = Self::normalize_agg_input_columns(
+ expr,
+ agg,
+ agg_input_has_derived_projection,
+ )?;
+ self.expr_to_sql(&expr)
+ })
.collect::<Result<Vec<_>>>()?,
vec![],
));
+ } else if Self::contains_aggregate_before_relation(agg.input.as_ref()) {
+ return self.derive_with_dialect_alias(
+ "derived_aggregate",
+ agg.input.as_ref(),
+ relation,
+ false,
+ vec![],
+ );
}
self.select_to_sql_recursively(
@@ -1992,11 +2093,10 @@ impl Unparser<'_> {
Ok(Some(relation))
}
- /// Strip the table qualifier from every column in a pushdown pass-through
- /// projection expression, so it resolves against the unnamed derived table
- /// rendered for the inner pushdown projection rather than a deeper table
- /// alias that is out of scope at this nesting level.
- fn strip_pushdown_column_qualifiers(expr: Expr) -> Result<Expr> {
+ /// Strip the table qualifier from every column in an expression that must
+ /// resolve against an unnamed derived table's output columns rather than a
+ /// deeper table alias that is out of scope at this nesting level.
+ fn strip_column_qualifiers(expr: Expr) -> Result<Expr> {
expr.transform(|e| match e {
Expr::Column(mut column) => {
column.relation = None;
@@ -2007,6 +2107,20 @@ impl Unparser<'_> {
.data()
}
+ fn strip_column_qualifiers_for_schema(expr: Expr, schema: &DFSchema) -> Result<Expr> {
+ expr.transform(|e| match e {
+ Expr::Column(mut column)
+ if column.relation.is_some()
+ && schema.index_of_column(&column).is_ok() =>
+ {
+ column.relation = None;
+ Ok(Transformed::yes(Expr::Column(column)))
+ }
+ other => Ok(Transformed::no(other)),
+ })
+ .data()
+ }
+
/// Try to unparse a table scan with pushdown operations into a new subquery plan.
/// If the table scan is without any pushdown operations, return None.
fn unparse_table_scan_pushdown(
@@ -2155,7 +2269,7 @@ impl Unparser<'_> {
.expr
.iter()
.cloned()
- .map(Self::strip_pushdown_column_qualifiers)
+ .map(Self::strip_column_qualifiers)
.collect::<Result<Vec<_>>>()?;
return Ok(Some(LogicalPlan::Projection(Projection::try_new(
exprs,