Apache DataFusion SQL Query Engine
fix: reject DISTINCT in corr, covar and regr_* instead of ignoring it (#26484)
Alejandro Borrego61ec8455dce2
diff --git a/datafusion/functions-aggregate/src/correlation.rs b/datafusion/functions-aggregate/src/correlation.rsindex e2877e63a1..f9110ccba7 100644--- a/datafusion/functions-aggregate/src/correlation.rs+++ b/datafusion/functions-aggregate/src/correlation.rs@@ -38,7 +38,7 @@ use log::debug;use crate::covariance::CovarianceAccumulator;use crate::stddev::StddevAccumulator;-use datafusion_common::{Result, ScalarValue};+use datafusion_common::{Result, ScalarValue, not_impl_err};use datafusion_expr::{Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,function::{AccumulatorArgs, StateFieldsArgs},@@ -108,7 +108,10 @@ impl AggregateUDFImpl for Correlation {Ok(DataType::Float64)}- fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {+ fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {+ if acc_args.is_distinct {+ return not_impl_err!("CORR(DISTINCT) aggregations are not available");+ }Ok(Box::new(CorrelationAccumulator::try_new()?))}@@ -139,8 +142,8 @@ impl AggregateUDFImpl for Correlation {self.doc()}- fn groups_accumulator_supported(&self, _args: AccumulatorArgs) -> bool {- true+ fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {+ !args.is_distinct}fn create_groups_accumulator(@@ -152,9 +155,9 @@ impl AggregateUDFImpl for Correlation {}fn distinct_handling(&self) -> DistinctHandling {- // Duplicate-sensitive, but the accumulator does not read- // `is_distinct` and today silently returns the non-distinct answer.- // The tag records the intent; enforcement is a follow-up change.+ // The accumulator rejects `DISTINCT` with `not_impl_err!`. It only+ // works when the planner removes it first (`SingleDistinctToGroupBy`+ // for a single distinct field, e.g. `corr(DISTINCT x, x)`).DistinctHandling::Unsupported}}diff --git a/datafusion/functions-aggregate/src/covariance.rs b/datafusion/functions-aggregate/src/covariance.rsindex 46abc82517..2477098728 100644--- a/datafusion/functions-aggregate/src/covariance.rs+++ b/datafusion/functions-aggregate/src/covariance.rs@@ -20,7 +20,7 @@use arrow::array::ArrayRef;use arrow::datatypes::{DataType, Field, FieldRef};use datafusion_common::cast::{as_float64_array, as_uint64_array};-use datafusion_common::{Result, ScalarValue};+use datafusion_common::{Result, ScalarValue, not_impl_err};use datafusion_expr::DistinctHandling;use datafusion_expr::{Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,@@ -118,7 +118,10 @@ impl AggregateUDFImpl for CovarianceSample {.collect())}- fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {+ fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {+ if acc_args.is_distinct {+ return not_impl_err!("COVAR_SAMP(DISTINCT) aggregations are not available");+ }Ok(Box::new(CovarianceAccumulator::try_new(StatsType::Sample)?))}@@ -131,9 +134,9 @@ impl AggregateUDFImpl for CovarianceSample {}fn distinct_handling(&self) -> DistinctHandling {- // Duplicate-sensitive, but the accumulator does not read- // `is_distinct` and today silently returns the non-distinct answer.- // The tag records the intent; enforcement is a follow-up change.+ // The accumulator rejects `DISTINCT` with `not_impl_err!`. It only+ // works when the planner removes it first (`SingleDistinctToGroupBy`+ // for a single distinct field, e.g. `covar_samp(DISTINCT x, x)`).DistinctHandling::Unsupported}}@@ -205,7 +208,10 @@ impl AggregateUDFImpl for CovariancePopulation {.collect())}- fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {+ fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {+ if acc_args.is_distinct {+ return not_impl_err!("COVAR_POP(DISTINCT) aggregations are not available");+ }Ok(Box::new(CovarianceAccumulator::try_new(StatsType::Population,)?))@@ -216,9 +222,9 @@ impl AggregateUDFImpl for CovariancePopulation {}fn distinct_handling(&self) -> DistinctHandling {- // Duplicate-sensitive, but the accumulator does not read- // `is_distinct` and today silently returns the non-distinct answer.- // The tag records the intent; enforcement is a follow-up change.+ // The accumulator rejects `DISTINCT` with `not_impl_err!`. It only+ // works when the planner removes it first (`SingleDistinctToGroupBy`+ // for a single distinct field, e.g. `covar_pop(DISTINCT x, x)`).DistinctHandling::Unsupported}}diff --git a/datafusion/functions-aggregate/src/regr.rs b/datafusion/functions-aggregate/src/regr.rsindex 6d679aabea..b6b4f2a7e4 100644--- a/datafusion/functions-aggregate/src/regr.rs+++ b/datafusion/functions-aggregate/src/regr.rs@@ -20,7 +20,7 @@use arrow::datatypes::FieldRef;use arrow::{array::ArrayRef, datatypes::DataType, datatypes::Field};use datafusion_common::cast::{as_float64_array, as_uint64_array};-use datafusion_common::{HashMap, Result, ScalarValue};+use datafusion_common::{HashMap, Result, ScalarValue, not_impl_err};use datafusion_doc::aggregate_doc_sections::DOC_SECTION_STATISTICAL;use datafusion_expr::DistinctHandling;use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};@@ -469,7 +469,13 @@ impl AggregateUDFImpl for Regr {self.regr_type != RegrType::Count}- fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {+ fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {+ if acc_args.is_distinct {+ return not_impl_err!(+ "{}(DISTINCT) aggregations are not available",+ self.func_name.to_uppercase()+ );+ }Ok(Box::new(RegrAccumulator::try_new(&self.regr_type)?))}@@ -516,9 +522,9 @@ impl AggregateUDFImpl for Regr {}fn distinct_handling(&self) -> DistinctHandling {- // Duplicate-sensitive, but the accumulator does not read- // `is_distinct` and today silently returns the non-distinct answer.- // The tag records the intent; enforcement is a follow-up change.+ // The accumulator rejects `DISTINCT` with `not_impl_err!`. It only+ // works when the planner removes it first (`SingleDistinctToGroupBy`+ // for a single distinct field, e.g. `regr_slope(DISTINCT x, x)`).DistinctHandling::Unsupported}}diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.sltindex c1eca4f223..356f0b705e 100644--- a/datafusion/sqllogictest/test_files/aggregate.slt+++ b/datafusion/sqllogictest/test_files/aggregate.slt@@ -432,6 +432,31 @@ GROUP BY g3 NULL4 NULL+# With two different arguments there is no single field to group by, so+# DISTINCT reaches the accumulator, which does not implement it.+statement error DataFusion error: This feature is not implemented: CORR\(DISTINCT\) aggregations are not available+SELECT corr(DISTINCT x, y) FROM (VALUES (1.0, 1.0), (1.0, 1.0), (2.0, 3.0)) t(x, y)++statement error DataFusion error: This feature is not implemented: CORR\(DISTINCT\) aggregations are not available+SELECT g, corr(DISTINCT x, y) FROM (VALUES (1, 1.0, 1.0), (1, 1.0, 1.0), (1, 2.0, 3.0)) t(g, x, y) GROUP BY g++statement error DataFusion error: This feature is not implemented: COVAR_SAMP\(DISTINCT\) aggregations are not available+SELECT covar_samp(DISTINCT x, y) FROM (VALUES (1.0, 1.0), (1.0, 1.0), (2.0, 3.0)) t(x, y)++statement error DataFusion error: This feature is not implemented: COVAR_POP\(DISTINCT\) aggregations are not available+SELECT covar_pop(x, y), covar_pop(DISTINCT x, y) FROM (VALUES (1.0, 1.0), (1.0, 1.0), (2.0, 3.0)) t(x, y)++statement error DataFusion error: This feature is not implemented: REGR_COUNT\(DISTINCT\) aggregations are not available+SELECT regr_count(x, y), regr_count(DISTINCT x, y) FROM (VALUES (1.0, 1.0), (1.0, 1.0), (2.0, 3.0)) t(x, y)++# Window functions build the accumulator through `create_sliding_accumulator`,+# which must reject DISTINCT too.+statement error DataFusion error: This feature is not implemented: CORR\(DISTINCT\) aggregations are not available+SELECT corr(DISTINCT x, y) OVER () FROM (VALUES (1.0, 1.0), (1.0, 1.0), (2.0, 3.0)) t(x, y)++statement error DataFusion error: This feature is not implemented: REGR_SLOPE\(DISTINCT\) aggregations are not available+SELECT regr_slope(DISTINCT x, y) OVER () FROM (VALUES (1.0, 1.0), (1.0, 1.0), (2.0, 3.0)) t(x, y)+# all_nulls_query_correlationquery Rwith data as (diff --git a/datafusion/sqllogictest/test_files/eliminate_join_distinct.slt b/datafusion/sqllogictest/test_files/eliminate_join_distinct.sltindex 006cad921f..eb9e034872 100644--- a/datafusion/sqllogictest/test_files/eliminate_join_distinct.slt+++ b/datafusion/sqllogictest/test_files/eliminate_join_distinct.slt@@ -187,11 +187,10 @@ SELECT COUNT(DISTINCT l.x), COUNT(l.g) FROM l JOIN r ON l.id = r.id;----2 7-# REGR_COUNT declares `DistinctHandling::Unsupported`: its accumulator ignores-# `is_distinct` and counts every joined row, so the results below are the-# non-distinct counts (4 and 6; the distinct counts would be 2 and 4). DISTINCT-# therefore does not hide the join fanout: the inner join must stay an inner-# join, and the left join must be kept.+# REGR_COUNT declares `DistinctHandling::Unsupported`: its accumulator rejects+# `is_distinct`, so the queries below fail instead of returning the non-distinct+# counts. DISTINCT therefore does not hide the join fanout: the inner join must+# stay an inner join, and the left join must be kept.query TTEXPLAIN SELECT REGR_COUNT(DISTINCT l.x, l.g) FROM l JOIN r ON l.id = r.id;----@@ -202,15 +201,11 @@ logical_plan04)------TableScan: l projection=[id, g, x]05)------TableScan: r projection=[id]-query I+statement error DataFusion error: This feature is not implemented: REGR_COUNT\(DISTINCT\) aggregations are not availableSELECT REGR_COUNT(DISTINCT l.x, l.g) FROM l JOIN r ON l.id = r.id;------4-query I+statement error DataFusion error: This feature is not implemented: REGR_COUNT\(DISTINCT\) aggregations are not availableSELECT REGR_COUNT(DISTINCT l.x, l.g) FROM l LEFT JOIN r ON l.id = r.id;------6# A limit below the aggregate still observes repeated join rows.query I