Skip to content

Apache DataFusion SQL Query Engine

fix: keep a DISTINCT aggregate's arity in `SingleDistinctToGroupBy` (#25461)

namanjain24-sudo6a31aa8d1dc4
diff --git a/datafusion/optimizer/src/single_distinct_to_groupby.rs b/datafusion/optimizer/src/single_distinct_to_groupby.rs
index f663003a9e..f3afcc8fee 100644
--- a/datafusion/optimizer/src/single_distinct_to_groupby.rs
+++ b/datafusion/optimizer/src/single_distinct_to_groupby.rs
@@ -23,7 +23,7 @@ use crate::optimizer::ApplyOrder;
use crate::{OptimizerConfig, OptimizerRule};
use datafusion_common::{
- DFSchema, DataFusionError, HashSet, Result, assert_eq_or_internal_err,
+ DFSchema, DataFusionError, HashSet, Result, assert_or_internal_err,
tree_node::Transformed,
};
use datafusion_expr::builder::project;
@@ -314,11 +314,18 @@ impl OptimizerRule for SingleDistinctToGroupBy {
},
}) => {
if distinct {
- assert_eq_or_internal_err!(
- args.len(),
- 1,
- "DISTINCT aggregate should have exactly one argument"
+ // `is_single_distinct_agg` checked that every
+ // distinct argument in the plan is the same
+ // expression. A call may still take it more than
+ // once, as `corr(DISTINCT x, x)` does, and keeps
+ // its arity here: its distinct argument tuples
+ // are the distinct values of that expression.
+ assert_or_internal_err!(
+ args.first()
+ .is_some_and(|f| args.iter().all(|a| a == f)),
+ "DISTINCT aggregate arguments should be the same expression"
);
+ let arity = args.len();
let arg = args.swap_remove(0);
if group_fields_set.insert(arg.schema_name().to_string())
@@ -329,7 +336,7 @@ impl OptimizerRule for SingleDistinctToGroupBy {
let outer =
Expr::AggregateFunction(AggregateFunction::new_udf(
func,
- vec![col(SINGLE_DISTINCT_ALIAS)],
+ vec![col(SINGLE_DISTINCT_ALIAS); arity],
false, // intentional to remove distinct here
filter,
order_by,
@@ -449,6 +456,7 @@ mod tests {
lit,
logical_plan::builder::{LogicalPlanBuilder, table_scan},
};
+ use datafusion_functions_aggregate::correlation::corr_udaf;
use datafusion_functions_aggregate::count::count_udaf;
use datafusion_functions_aggregate::expr_fn::{count, count_distinct, max, min, sum};
use datafusion_functions_aggregate::min_max::max_udaf;
@@ -465,6 +473,17 @@ mod tests {
))
}
+ fn corr_distinct(y: Expr, x: Expr) -> Expr {
+ Expr::AggregateFunction(AggregateFunction::new_udf(
+ corr_udaf(),
+ vec![y, x],
+ true,
+ None,
+ vec![],
+ None,
+ ))
+ }
+
fn sum_distinct(expr: Expr) -> Expr {
Expr::AggregateFunction(AggregateFunction::new_udf(
sum_udaf(),
@@ -770,6 +789,46 @@ mod tests {
)
}
+ #[test]
+ fn single_distinct_repeated_arg_and_groupby() -> Result<()> {
+ let table_scan = test_table_scan()?;
+
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .aggregate(vec![col("a")], vec![corr_distinct(col("b"), col("b"))])?
+ .build()?;
+
+ // Should work: the call takes the distinct argument twice, so it keeps
+ // both arguments after the rewrite.
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Projection: test.a, corr(alias1,alias1) AS corr(DISTINCT test.b,test.b) [a:UInt32, corr(DISTINCT test.b,test.b):Float64;N]
+ Aggregate: groupBy=[[test.a]], aggr=[[corr(alias1, alias1)]] [a:UInt32, corr(alias1,alias1):Float64;N]
+ Aggregate: groupBy=[[test.a, test.b AS alias1]], aggr=[[]] [a:UInt32, alias1:UInt32]
+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]
+ "
+ )
+ }
+
+ #[test]
+ fn single_distinct_two_args_and_groupby() -> Result<()> {
+ let table_scan = test_table_scan()?;
+
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .aggregate(vec![col("a")], vec![corr_distinct(col("b"), col("c"))])?
+ .build()?;
+
+ // Do nothing: the arguments are different expressions, so there is no
+ // single field to group by.
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Aggregate: groupBy=[[test.a]], aggr=[[corr(DISTINCT test.b, test.c)]] [a:UInt32, corr(DISTINCT test.b,test.c):Float64;N]
+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]
+ "
+ )
+ }
+
#[test]
fn one_field_two_distinct_and_groupby() -> Result<()> {
let table_scan = test_table_scan()?;
diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt
index 61e4bd2543..ee341880f7 100644
--- a/datafusion/sqllogictest/test_files/aggregate.slt
+++ b/datafusion/sqllogictest/test_files/aggregate.slt
@@ -404,6 +404,34 @@ GROUP BY g
1 1
2 NULL
+# A DISTINCT aggregate may take the distinct argument more than once.
+# Group 1 aggregates over the distinct values 1.0 and 2.0, group 2 over the
+# single distinct value 3.0, which leaves the sample covariance undefined.
+# Group 3 repeats a NULL row alongside the distinct value 2.0: DISTINCT
+# collapses the repeated NULLs into one, and the aggregate then excludes
+# that NULL the same way it would without the rewrite, leaving a single
+# distinct point - again an undefined sample covariance. Group 4 is all
+# NULL, so the aggregate has zero non-null distinct points to work with.
+query IR rowsort
+SELECT g, corr(DISTINCT x, x)
+FROM (VALUES (1, 1.0), (1, 1.0), (1, 2.0), (2, 3.0), (2, 3.0), (3, NULL), (3, NULL), (3, 2.0), (4, NULL), (4, NULL)) AS t(g, x)
+GROUP BY g
+----
+1 1
+2 NULL
+3 NULL
+4 NULL
+
+query IR rowsort
+SELECT g, covar_samp(DISTINCT x, x)
+FROM (VALUES (1, 1.0), (1, 1.0), (1, 2.0), (2, 3.0), (2, 3.0), (3, NULL), (3, NULL), (3, 2.0), (4, NULL), (4, NULL)) AS t(g, x)
+GROUP BY g
+----
+1 0.5
+2 NULL
+3 NULL
+4 NULL
+
# all_nulls_query_correlation
query R
with data as (