Skip to content

Apache DataFusion SQL Query Engine

Fix panic from `avg(x ORDER BY y)` (and more) (#25711)

Masone8e41ae958a5
diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rs
index 74f116cf48..a54a382d89 100644
--- a/datafusion/functions-aggregate/src/approx_distinct.rs
+++ b/datafusion/functions-aggregate/src/approx_distinct.rs
@@ -40,7 +40,7 @@ use datafusion_common::{
};
use datafusion_expr::DistinctHandling;
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
-use datafusion_expr::utils::format_state_name;
+use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Documentation, EmitTo, GroupsAccumulator, Signature,
Volatility,
@@ -868,6 +868,10 @@ impl AggregateUDFImpl for ApproxDistinct {
}
}
+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {
+ AggregateOrderSensitivity::Insensitive
+ }
+
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
diff --git a/datafusion/functions-aggregate/src/average.rs b/datafusion/functions-aggregate/src/average.rs
index f59ac816a9..85b63a9a17 100644
--- a/datafusion/functions-aggregate/src/average.rs
+++ b/datafusion/functions-aggregate/src/average.rs
@@ -36,7 +36,7 @@ use datafusion_common::{
Result, ScalarValue, exec_datafusion_err, exec_err, internal_err, not_impl_err,
};
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
-use datafusion_expr::utils::format_state_name;
+use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Coercion, Documentation, EmitTo, Expr, GroupSelection,
GroupsAccumulator, ReversedUDAF, Signature, TypeSignature, TypeSignatureClass,
@@ -492,6 +492,10 @@ impl AggregateUDFImpl for Avg {
ReversedUDAF::Identical
}
+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {
+ AggregateOrderSensitivity::Insensitive
+ }
+
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
diff --git a/datafusion/functions-aggregate/src/bit_and_or_xor.rs b/datafusion/functions-aggregate/src/bit_and_or_xor.rs
index 02d77c84ba..477ad6f1ef 100644
--- a/datafusion/functions-aggregate/src/bit_and_or_xor.rs
+++ b/datafusion/functions-aggregate/src/bit_and_or_xor.rs
@@ -33,7 +33,7 @@ use datafusion_common::cast::as_list_array;
use datafusion_common::{Result, ScalarValue, not_impl_err};
use datafusion_expr::DistinctHandling;
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
-use datafusion_expr::utils::format_state_name;
+use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Coercion, Documentation, GroupsAccumulator,
ReversedUDAF, Signature, TypeSignatureClass, Volatility,
@@ -316,6 +316,10 @@ impl AggregateUDFImpl for BitwiseOperation {
ReversedUDAF::Identical
}
+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {
+ AggregateOrderSensitivity::Insensitive
+ }
+
fn documentation(&self) -> Option<&Documentation> {
Some(self.documentation)
}
diff --git a/datafusion/functions-aggregate/src/correlation.rs b/datafusion/functions-aggregate/src/correlation.rs
index c3832f8de6..e2877e63a1 100644
--- a/datafusion/functions-aggregate/src/correlation.rs
+++ b/datafusion/functions-aggregate/src/correlation.rs
@@ -42,7 +42,7 @@ use datafusion_common::{Result, ScalarValue};
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,
function::{AccumulatorArgs, StateFieldsArgs},
- utils::format_state_name,
+ utils::{AggregateOrderSensitivity, format_state_name},
};
use datafusion_functions_aggregate_common::stats::StatsType;
use datafusion_macros::user_doc;
@@ -131,6 +131,10 @@ impl AggregateUDFImpl for Correlation {
.collect())
}
+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {
+ AggregateOrderSensitivity::Insensitive
+ }
+
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
diff --git a/datafusion/functions-aggregate/src/median.rs b/datafusion/functions-aggregate/src/median.rs
index d2db50f374..c96baddfee 100644
--- a/datafusion/functions-aggregate/src/median.rs
+++ b/datafusion/functions-aggregate/src/median.rs
@@ -28,6 +28,7 @@ use datafusion_common::Result;
use datafusion_common::types::logical_float64;
use datafusion_expr::GroupsAccumulator;
use datafusion_expr::function::StateFieldsArgs;
+use datafusion_expr::utils::AggregateOrderSensitivity;
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Coercion, Documentation, Signature, TypeSignature,
TypeSignatureClass, Volatility, function::AccumulatorArgs,
@@ -143,6 +144,10 @@ impl AggregateUDFImpl for Median {
)
}
+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {
+ AggregateOrderSensitivity::Insensitive
+ }
+
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
diff --git a/datafusion/functions-aggregate/src/stddev.rs b/datafusion/functions-aggregate/src/stddev.rs
index 14a28fa809..b9f9440690 100644
--- a/datafusion/functions-aggregate/src/stddev.rs
+++ b/datafusion/functions-aggregate/src/stddev.rs
@@ -28,7 +28,7 @@ use arrow::{array::ArrayRef, datatypes::DataType, datatypes::Field};
use datafusion_common::ScalarValue;
use datafusion_common::{Result, internal_err, not_impl_err};
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
-use datafusion_expr::utils::format_state_name;
+use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Documentation, GroupSelection, GroupsAccumulator,
Signature, Volatility,
@@ -137,6 +137,10 @@ impl AggregateUDFImpl for Stddev {
Ok(Box::new(StddevGroupsAccumulator::new(StatsType::Sample)))
}
+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {
+ AggregateOrderSensitivity::Insensitive
+ }
+
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
@@ -243,6 +247,10 @@ impl AggregateUDFImpl for StddevPop {
)))
}
+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {
+ AggregateOrderSensitivity::Insensitive
+ }
+
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
diff --git a/datafusion/functions-aggregate/src/variance.rs b/datafusion/functions-aggregate/src/variance.rs
index 154a677cb2..6da00aa5d5 100644
--- a/datafusion/functions-aggregate/src/variance.rs
+++ b/datafusion/functions-aggregate/src/variance.rs
@@ -30,7 +30,7 @@ use datafusion_expr::{
Accumulator, AggregateUDFImpl, Documentation, GroupSelection, GroupsAccumulator,
Signature, Volatility,
function::{AccumulatorArgs, StateFieldsArgs},
- utils::format_state_name,
+ utils::{AggregateOrderSensitivity, format_state_name},
};
use datafusion_functions_aggregate_common::utils::GenericDistinctBuffer;
use datafusion_functions_aggregate_common::{
@@ -147,6 +147,10 @@ impl AggregateUDFImpl for VarianceSample {
Ok(Box::new(VarianceGroupsAccumulator::new(StatsType::Sample)))
}
+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {
+ AggregateOrderSensitivity::Insensitive
+ }
+
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
@@ -252,6 +256,10 @@ impl AggregateUDFImpl for VariancePopulation {
)))
}
+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {
+ AggregateOrderSensitivity::Insensitive
+ }
+
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
diff --git a/datafusion/spark/src/function/aggregate/avg.rs b/datafusion/spark/src/function/aggregate/avg.rs
index 46e63013db..38778e49be 100644
--- a/datafusion/spark/src/function/aggregate/avg.rs
+++ b/datafusion/spark/src/function/aggregate/avg.rs
@@ -27,7 +27,7 @@ use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::types::{NativeType, logical_float64};
use datafusion_common::{Result, ScalarValue, not_impl_err};
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
-use datafusion_expr::utils::format_state_name;
+use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Coercion, EmitTo, GroupsAccumulator, ReversedUDAF,
Signature, TypeSignatureClass, Volatility,
@@ -117,6 +117,10 @@ impl AggregateUDFImpl for SparkAvg {
ReversedUDAF::Identical
}
+ fn order_sensitivity(&self) -> AggregateOrderSensitivity {
+ AggregateOrderSensitivity::Insensitive
+ }
+
fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {
!args.is_distinct
}
diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt
index 2be4e173fc..b3f8a5515d 100644
--- a/datafusion/sqllogictest/test_files/aggregate.slt
+++ b/datafusion/sqllogictest/test_files/aggregate.slt
@@ -9951,7 +9951,9 @@ SELECT sum(s) FROM (SELECT sum(column2) OVER () AS s FROM nested_agg_t);
statement ok
DROP TABLE nested_agg_t;
-# Tests for COUNT(... ORDER BY ...)
+# Tests for aggregates that ignore ORDER BY
+
+# COUNT(... ORDER BY ...)
statement ok
CREATE TABLE count_order_by_t (a INT, b INT, c INT) AS VALUES
(1, NULL, 10),
@@ -9993,3 +9995,89 @@ physical_plan
statement ok
DROP TABLE count_order_by_t;
+
+# Aggregates that keep the default `order_sensitivity` receive the ORDER BY
+# columns as extra arguments, which their GroupsAccumulator must ignore.
+statement ok
+CREATE TABLE agg_order_by_t (g INT, v DOUBLE, i BIGINT) AS VALUES
+(1, 1.0, 1),
+(1, 3.0, 3),
+(2, 5.0, 5),
+(2, 7.0, 7),
+(2, 9.0, 9);
+
+# Same aggregates without ORDER BY, for comparison
+query IRIR
+SELECT g, avg(v), bit_and(i), var(v) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 2 1 2
+2 7 1 4
+
+query IR
+SELECT g, avg(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 2
+2 7
+
+query II
+SELECT g, bit_and(i ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 1
+2 1
+
+query II
+SELECT g, bit_or(i ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 3
+2 15
+
+query II
+SELECT g, bit_xor(i ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 2
+2 11
+
+query IR
+SELECT g, var(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 2
+2 4
+
+query IR
+SELECT g, var_pop(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 1
+2 2.666666666667
+
+query IR
+SELECT g, stddev(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 1.414213562373
+2 2
+
+query IR
+SELECT g, stddev_pop(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 1
+2 1.632993161855
+
+# DISTINCT uses a different Accumulator
+query IRR
+SELECT g, avg(DISTINCT v ORDER BY v), var(DISTINCT v ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 2 2
+2 7 4
+
+# Without GROUP BY the Accumulator is used instead of the GroupsAccumulator
+query R
+SELECT avg(v ORDER BY i) FROM agg_order_by_t;
+----
+5
+
+query RR
+SELECT avg(DISTINCT v ORDER BY v), var(DISTINCT v ORDER BY v) FROM agg_order_by_t;
+----
+5 10
+
+statement ok
+DROP TABLE agg_order_by_t;
diff --git a/datafusion/sqllogictest/test_files/aggregate_skip_partial.slt b/datafusion/sqllogictest/test_files/aggregate_skip_partial.slt
index 26ba9aebc3..d659e4c836 100644
--- a/datafusion/sqllogictest/test_files/aggregate_skip_partial.slt
+++ b/datafusion/sqllogictest/test_files/aggregate_skip_partial.slt
@@ -823,3 +823,74 @@ reset datafusion.execution.skip_partial_aggregation_probe_rows_threshold;
statement ok
DROP TABLE aggregate_test_100_bool
+
+# `convert_to_state` receives every argument, so aggregates that ignore
+# ORDER BY must not pass the ORDER BY columns through to it
+statement ok
+set datafusion.execution.skip_partial_aggregation_probe_rows_threshold = 0;
+
+statement ok
+set datafusion.execution.skip_partial_aggregation_probe_ratio_threshold = 0.0;
+
+statement ok
+set datafusion.execution.target_partitions = 2;
+
+statement ok
+set datafusion.execution.batch_size = 1;
+
+statement ok
+CREATE TABLE agg_order_by_t (g INT, v DOUBLE, i BIGINT) AS VALUES
+(1, 1.0, 1),
+(1, 3.0, 3),
+(2, 5.0, 5),
+(2, 7.0, 7),
+(2, 9.0, 9);
+
+# median without ORDER BY, for comparison
+query IR
+SELECT g, median(v) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 2
+2 7
+
+query IR
+SELECT g, median(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 2
+2 7
+
+query II
+SELECT g, approx_distinct(i ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 2
+2 3
+
+query IR
+SELECT g, corr(v, i ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 1
+2 1
+
+query IR
+SELECT g, var(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
+----
+1 2
+2 4
+
+statement ok
+DROP TABLE agg_order_by_t;
+
+# Config reset
+statement ok
+reset datafusion.execution.batch_size;
+
+# The SLT runner sets `target_partitions` to 4 instead of using the default, so
+# reset it explicitly.
+statement ok
+set datafusion.execution.target_partitions = 4;
+
+statement ok
+reset datafusion.execution.skip_partial_aggregation_probe_ratio_threshold;
+
+statement ok
+reset datafusion.execution.skip_partial_aggregation_probe_rows_threshold;
diff --git a/datafusion/sqllogictest/test_files/spark/aggregate/avg.slt b/datafusion/sqllogictest/test_files/spark/aggregate/avg.slt
index afbd5fe2e5..eee84e9f36 100644
--- a/datafusion/sqllogictest/test_files/spark/aggregate/avg.slt
+++ b/datafusion/sqllogictest/test_files/spark/aggregate/avg.slt
@@ -54,3 +54,13 @@ GROUP BY a
ORDER BY a;
----
0 0
+
+# avg ignores ORDER BY, so the ORDER BY column must not reach its accumulators
+query IR
+SELECT a % 2 AS g, avg(a ORDER BY b)
+FROM (VALUES (40, 1), (23, 2), (17, 3), (40, 4), (30, 5)) AS t(a, b)
+GROUP BY g
+ORDER BY g;
+----
+0 36.666666666666664
+1 20