Apache DataFusion SQL Query Engine
Fix panic from `avg(x ORDER BY y)` (and more) (#25711)
Masone8e41ae958a5
diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rsindex 74f116cf48..a54a382d89 100644--- a/datafusion/functions-aggregate/src/approx_distinct.rs+++ b/datafusion/functions-aggregate/src/approx_distinct.rs@@ -40,7 +40,7 @@ use datafusion_common::{};use datafusion_expr::DistinctHandling;use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};-use datafusion_expr::utils::format_state_name;+use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};use datafusion_expr::{Accumulator, AggregateUDFImpl, Documentation, EmitTo, GroupsAccumulator, Signature,Volatility,@@ -868,6 +868,10 @@ impl AggregateUDFImpl for ApproxDistinct {}}+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {+ AggregateOrderSensitivity::Insensitive+ }+fn documentation(&self) -> Option<&Documentation> {self.doc()}diff --git a/datafusion/functions-aggregate/src/average.rs b/datafusion/functions-aggregate/src/average.rsindex f59ac816a9..85b63a9a17 100644--- a/datafusion/functions-aggregate/src/average.rs+++ b/datafusion/functions-aggregate/src/average.rs@@ -36,7 +36,7 @@ use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err, internal_err, not_impl_err,};use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};-use datafusion_expr::utils::format_state_name;+use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};use datafusion_expr::{Accumulator, AggregateUDFImpl, Coercion, Documentation, EmitTo, Expr, GroupSelection,GroupsAccumulator, ReversedUDAF, Signature, TypeSignature, TypeSignatureClass,@@ -492,6 +492,10 @@ impl AggregateUDFImpl for Avg {ReversedUDAF::Identical}+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {+ AggregateOrderSensitivity::Insensitive+ }+fn documentation(&self) -> Option<&Documentation> {self.doc()}diff --git a/datafusion/functions-aggregate/src/bit_and_or_xor.rs b/datafusion/functions-aggregate/src/bit_and_or_xor.rsindex 02d77c84ba..477ad6f1ef 100644--- a/datafusion/functions-aggregate/src/bit_and_or_xor.rs+++ b/datafusion/functions-aggregate/src/bit_and_or_xor.rs@@ -33,7 +33,7 @@ use datafusion_common::cast::as_list_array;use datafusion_common::{Result, ScalarValue, not_impl_err};use datafusion_expr::DistinctHandling;use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};-use datafusion_expr::utils::format_state_name;+use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};use datafusion_expr::{Accumulator, AggregateUDFImpl, Coercion, Documentation, GroupsAccumulator,ReversedUDAF, Signature, TypeSignatureClass, Volatility,@@ -316,6 +316,10 @@ impl AggregateUDFImpl for BitwiseOperation {ReversedUDAF::Identical}+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {+ AggregateOrderSensitivity::Insensitive+ }+fn documentation(&self) -> Option<&Documentation> {Some(self.documentation)}diff --git a/datafusion/functions-aggregate/src/correlation.rs b/datafusion/functions-aggregate/src/correlation.rsindex c3832f8de6..e2877e63a1 100644--- a/datafusion/functions-aggregate/src/correlation.rs+++ b/datafusion/functions-aggregate/src/correlation.rs@@ -42,7 +42,7 @@ use datafusion_common::{Result, ScalarValue};use datafusion_expr::{Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,function::{AccumulatorArgs, StateFieldsArgs},- utils::format_state_name,+ utils::{AggregateOrderSensitivity, format_state_name},};use datafusion_functions_aggregate_common::stats::StatsType;use datafusion_macros::user_doc;@@ -131,6 +131,10 @@ impl AggregateUDFImpl for Correlation {.collect())}+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {+ AggregateOrderSensitivity::Insensitive+ }+fn documentation(&self) -> Option<&Documentation> {self.doc()}diff --git a/datafusion/functions-aggregate/src/median.rs b/datafusion/functions-aggregate/src/median.rsindex d2db50f374..c96baddfee 100644--- a/datafusion/functions-aggregate/src/median.rs+++ b/datafusion/functions-aggregate/src/median.rs@@ -28,6 +28,7 @@ use datafusion_common::Result;use datafusion_common::types::logical_float64;use datafusion_expr::GroupsAccumulator;use datafusion_expr::function::StateFieldsArgs;+use datafusion_expr::utils::AggregateOrderSensitivity;use datafusion_expr::{Accumulator, AggregateUDFImpl, Coercion, Documentation, Signature, TypeSignature,TypeSignatureClass, Volatility, function::AccumulatorArgs,@@ -143,6 +144,10 @@ impl AggregateUDFImpl for Median {)}+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {+ AggregateOrderSensitivity::Insensitive+ }+fn documentation(&self) -> Option<&Documentation> {self.doc()}diff --git a/datafusion/functions-aggregate/src/stddev.rs b/datafusion/functions-aggregate/src/stddev.rsindex 14a28fa809..b9f9440690 100644--- a/datafusion/functions-aggregate/src/stddev.rs+++ b/datafusion/functions-aggregate/src/stddev.rs@@ -28,7 +28,7 @@ use arrow::{array::ArrayRef, datatypes::DataType, datatypes::Field};use datafusion_common::ScalarValue;use datafusion_common::{Result, internal_err, not_impl_err};use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};-use datafusion_expr::utils::format_state_name;+use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};use datafusion_expr::{Accumulator, AggregateUDFImpl, Documentation, GroupSelection, GroupsAccumulator,Signature, Volatility,@@ -137,6 +137,10 @@ impl AggregateUDFImpl for Stddev {Ok(Box::new(StddevGroupsAccumulator::new(StatsType::Sample)))}+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {+ AggregateOrderSensitivity::Insensitive+ }+fn documentation(&self) -> Option<&Documentation> {self.doc()}@@ -243,6 +247,10 @@ impl AggregateUDFImpl for StddevPop {)))}+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {+ AggregateOrderSensitivity::Insensitive+ }+fn documentation(&self) -> Option<&Documentation> {self.doc()}diff --git a/datafusion/functions-aggregate/src/variance.rs b/datafusion/functions-aggregate/src/variance.rsindex 154a677cb2..6da00aa5d5 100644--- a/datafusion/functions-aggregate/src/variance.rs+++ b/datafusion/functions-aggregate/src/variance.rs@@ -30,7 +30,7 @@ use datafusion_expr::{Accumulator, AggregateUDFImpl, Documentation, GroupSelection, GroupsAccumulator,Signature, Volatility,function::{AccumulatorArgs, StateFieldsArgs},- utils::format_state_name,+ utils::{AggregateOrderSensitivity, format_state_name},};use datafusion_functions_aggregate_common::utils::GenericDistinctBuffer;use datafusion_functions_aggregate_common::{@@ -147,6 +147,10 @@ impl AggregateUDFImpl for VarianceSample {Ok(Box::new(VarianceGroupsAccumulator::new(StatsType::Sample)))}+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {+ AggregateOrderSensitivity::Insensitive+ }+fn documentation(&self) -> Option<&Documentation> {self.doc()}@@ -252,6 +256,10 @@ impl AggregateUDFImpl for VariancePopulation {)))}+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {+ AggregateOrderSensitivity::Insensitive+ }+fn documentation(&self) -> Option<&Documentation> {self.doc()}diff --git a/datafusion/spark/src/function/aggregate/avg.rs b/datafusion/spark/src/function/aggregate/avg.rsindex 46e63013db..38778e49be 100644--- a/datafusion/spark/src/function/aggregate/avg.rs+++ b/datafusion/spark/src/function/aggregate/avg.rs@@ -27,7 +27,7 @@ use arrow::datatypes::{DataType, Field, FieldRef};use datafusion_common::types::{NativeType, logical_float64};use datafusion_common::{Result, ScalarValue, not_impl_err};use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};-use datafusion_expr::utils::format_state_name;+use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};use datafusion_expr::{Accumulator, AggregateUDFImpl, Coercion, EmitTo, GroupsAccumulator, ReversedUDAF,Signature, TypeSignatureClass, Volatility,@@ -117,6 +117,10 @@ impl AggregateUDFImpl for SparkAvg {ReversedUDAF::Identical}+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {+ AggregateOrderSensitivity::Insensitive+ }+fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {!args.is_distinct}diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.sltindex 2be4e173fc..b3f8a5515d 100644--- a/datafusion/sqllogictest/test_files/aggregate.slt+++ b/datafusion/sqllogictest/test_files/aggregate.slt@@ -9951,7 +9951,9 @@ SELECT sum(s) FROM (SELECT sum(column2) OVER () AS s FROM nested_agg_t);statement okDROP TABLE nested_agg_t;-# Tests for COUNT(... ORDER BY ...)+# Tests for aggregates that ignore ORDER BY++# COUNT(... ORDER BY ...)statement okCREATE TABLE count_order_by_t (a INT, b INT, c INT) AS VALUES(1, NULL, 10),@@ -9993,3 +9995,89 @@ physical_planstatement okDROP TABLE count_order_by_t;++# Aggregates that keep the default `order_sensitivity` receive the ORDER BY+# columns as extra arguments, which their GroupsAccumulator must ignore.+statement ok+CREATE TABLE agg_order_by_t (g INT, v DOUBLE, i BIGINT) AS VALUES+(1, 1.0, 1),+(1, 3.0, 3),+(2, 5.0, 5),+(2, 7.0, 7),+(2, 9.0, 9);++# Same aggregates without ORDER BY, for comparison+query IRIR+SELECT g, avg(v), bit_and(i), var(v) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 2 1 2+2 7 1 4++query IR+SELECT g, avg(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 2+2 7++query II+SELECT g, bit_and(i ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 1+2 1++query II+SELECT g, bit_or(i ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 3+2 15++query II+SELECT g, bit_xor(i ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 2+2 11++query IR+SELECT g, var(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 2+2 4++query IR+SELECT g, var_pop(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 1+2 2.666666666667++query IR+SELECT g, stddev(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 1.414213562373+2 2++query IR+SELECT g, stddev_pop(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 1+2 1.632993161855++# DISTINCT uses a different Accumulator+query IRR+SELECT g, avg(DISTINCT v ORDER BY v), var(DISTINCT v ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 2 2+2 7 4++# Without GROUP BY the Accumulator is used instead of the GroupsAccumulator+query R+SELECT avg(v ORDER BY i) FROM agg_order_by_t;+----+5++query RR+SELECT avg(DISTINCT v ORDER BY v), var(DISTINCT v ORDER BY v) FROM agg_order_by_t;+----+5 10++statement ok+DROP TABLE agg_order_by_t;diff --git a/datafusion/sqllogictest/test_files/aggregate_skip_partial.slt b/datafusion/sqllogictest/test_files/aggregate_skip_partial.sltindex 26ba9aebc3..d659e4c836 100644--- a/datafusion/sqllogictest/test_files/aggregate_skip_partial.slt+++ b/datafusion/sqllogictest/test_files/aggregate_skip_partial.slt@@ -823,3 +823,74 @@ reset datafusion.execution.skip_partial_aggregation_probe_rows_threshold;statement okDROP TABLE aggregate_test_100_bool++# `convert_to_state` receives every argument, so aggregates that ignore+# ORDER BY must not pass the ORDER BY columns through to it+statement ok+set datafusion.execution.skip_partial_aggregation_probe_rows_threshold = 0;++statement ok+set datafusion.execution.skip_partial_aggregation_probe_ratio_threshold = 0.0;++statement ok+set datafusion.execution.target_partitions = 2;++statement ok+set datafusion.execution.batch_size = 1;++statement ok+CREATE TABLE agg_order_by_t (g INT, v DOUBLE, i BIGINT) AS VALUES+(1, 1.0, 1),+(1, 3.0, 3),+(2, 5.0, 5),+(2, 7.0, 7),+(2, 9.0, 9);++# median without ORDER BY, for comparison+query IR+SELECT g, median(v) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 2+2 7++query IR+SELECT g, median(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 2+2 7++query II+SELECT g, approx_distinct(i ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 2+2 3++query IR+SELECT g, corr(v, i ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 1+2 1++query IR+SELECT g, var(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;+----+1 2+2 4++statement ok+DROP TABLE agg_order_by_t;++# Config reset+statement ok+reset datafusion.execution.batch_size;++# The SLT runner sets `target_partitions` to 4 instead of using the default, so+# reset it explicitly.+statement ok+set datafusion.execution.target_partitions = 4;++statement ok+reset datafusion.execution.skip_partial_aggregation_probe_ratio_threshold;++statement ok+reset datafusion.execution.skip_partial_aggregation_probe_rows_threshold;diff --git a/datafusion/sqllogictest/test_files/spark/aggregate/avg.slt b/datafusion/sqllogictest/test_files/spark/aggregate/avg.sltindex afbd5fe2e5..eee84e9f36 100644--- a/datafusion/sqllogictest/test_files/spark/aggregate/avg.slt+++ b/datafusion/sqllogictest/test_files/spark/aggregate/avg.slt@@ -54,3 +54,13 @@ GROUP BY aORDER BY a;----0 0++# avg ignores ORDER BY, so the ORDER BY column must not reach its accumulators+query IR+SELECT a % 2 AS g, avg(a ORDER BY b)+FROM (VALUES (40, 1), (23, 2), (17, 3), (40, 4), (30, 5)) AS t(a, b)+GROUP BY g+ORDER BY g;+----+0 36.666666666666664+1 20