Apache DataFusion SQL Query Engine
fix: evaluate grouped aggregate arguments after FILTER (#24444)
linfeng5610e58b9163
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rsindex f7c12b1931..6aa246beed 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs@@ -179,9 +179,7 @@ impl<AggrMode> AggregateHashTable<AggrMode> {let timer = self.group_by_metrics.aggregate_arguments_time.timer();// The evaluated args for each accumulator- let accumulator_args = self- .state- .building()+ let accumulator_args = state.accumulators.iter().enumerate()@@ -562,23 +560,16 @@ impl HashAggregateAccumulator {/// Evaluate aggregate arguments and filter for one input batch.///- /// For example, `AVG(x + 1) FILTER (WHERE x > 0)` evaluates both `x + 1`- /// and `x > 0`.+ /// For example, `AVG(2 / x) FILTER (WHERE x > 0)` evaluates `x > 0`+ /// first, then evaluates `2 / x` only for selected rows.+ /// Filtered rows will be evaluated to `NULL`, and won't trigger errors+ /// such as divide by zero.////// These arrays can be passed directly to [`GroupsAccumulator`] next.pub(super) fn evaluate_acc_args(&self,batch: &RecordBatch,) -> Result<EvaluatedAccumulatorArgs> {- let arguments = self- .arguments- .iter()- .map(|expr| {- expr.evaluate(batch)- .and_then(|value| value.into_array(batch.num_rows()))- })- .collect::<Result<_>>()?;-let filter = self.filter.as_ref()@@ -588,6 +579,19 @@ impl HashAggregateAccumulator {.and_then(|value| value.into_array(batch.num_rows()))}).transpose()?;+ let selection = filter.as_ref().map(|filter| filter.as_boolean());+ let arguments = self+ .arguments+ .iter()+ .map(|expr| {+ selection+ .map_or_else(+ || expr.evaluate(batch),+ |selection| expr.evaluate_selection(batch, selection),+ )+ .and_then(|value| value.into_array(batch.num_rows()))+ })+ .collect::<Result<_>>()?;Ok(EvaluatedAccumulatorArgs { arguments, filter })}diff --git a/datafusion/sqllogictest/test_files/aggregate_filter_selection.slt b/datafusion/sqllogictest/test_files/aggregate_filter_selection.sltnew file mode 100644index 0000000000..ca58378584--- /dev/null+++ b/datafusion/sqllogictest/test_files/aggregate_filter_selection.slt@@ -0,0 +1,100 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at++# http://www.apache.org/licenses/LICENSE-2.0++# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# Aggregate FILTER must select rows before evaluating fallible arguments.++statement ok+CREATE TABLE aggregate_filter_selection (g INT, v BIGINT);++statement ok+INSERT INTO aggregate_filter_selection VALUES+ (1, 0),+ (2, 2),+ (3, 0),+ (4, 5);++statement ok+SET datafusion.execution.skip_partial_aggregation_probe_rows_threshold = 100000;++statement ok+SET datafusion.execution.enable_migration_aggregate = true;++query II+SELECT g, SUM(10 / v) FILTER (WHERE v <> 0)+FROM aggregate_filter_selection+GROUP BY g+ORDER BY g;+----+1 NULL+2 5+3 NULL+4 2++statement ok+SET datafusion.execution.target_partitions = 2;++statement ok+SET datafusion.execution.batch_size = 1;++statement ok+SET datafusion.execution.skip_partial_aggregation_probe_rows_threshold = 0;++statement ok+SET datafusion.execution.skip_partial_aggregation_probe_ratio_threshold = 0.0;++statement ok+CREATE TABLE aggregate_filter_selection_skip (g INT, v BIGINT);++statement ok+INSERT INTO aggregate_filter_selection_skip VALUES+ (1, 2),+ (3, 5),+ (2, 0);++statement ok+SET datafusion.execution.enable_migration_aggregate = true;++query II+SELECT g, SUM(10 / v) FILTER (WHERE v <> 0)+FROM aggregate_filter_selection_skip+GROUP BY g+ORDER BY g;+----+1 5+2 NULL+3 2++statement ok+RESET datafusion.execution.enable_migration_aggregate;++statement ok+RESET datafusion.execution.skip_partial_aggregation_probe_ratio_threshold;++statement ok+RESET datafusion.execution.skip_partial_aggregation_probe_rows_threshold;++statement ok+RESET datafusion.execution.batch_size;++statement ok+SET datafusion.execution.target_partitions = 4;++statement ok+DROP TABLE aggregate_filter_selection_skip;++statement ok+DROP TABLE aggregate_filter_selection;