Skip to content

Apache DataFusion SQL Query Engine

fix: evaluate grouped aggregate arguments after FILTER (#24444)

linfeng5610e58b9163
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs
index f7c12b1931..6aa246beed 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs
@@ -179,9 +179,7 @@ impl<AggrMode> AggregateHashTable<AggrMode> {
let timer = self.group_by_metrics.aggregate_arguments_time.timer();
// The evaluated args for each accumulator
- let accumulator_args = self
- .state
- .building()
+ let accumulator_args = state
.accumulators
.iter()
.enumerate()
@@ -562,23 +560,16 @@ impl HashAggregateAccumulator {
/// Evaluate aggregate arguments and filter for one input batch.
///
- /// For example, `AVG(x + 1) FILTER (WHERE x > 0)` evaluates both `x + 1`
- /// and `x > 0`.
+ /// For example, `AVG(2 / x) FILTER (WHERE x > 0)` evaluates `x > 0`
+ /// first, then evaluates `2 / x` only for selected rows.
+ /// Filtered rows will be evaluated to `NULL`, and won't trigger errors
+ /// such as divide by zero.
///
/// These arrays can be passed directly to [`GroupsAccumulator`] next.
pub(super) fn evaluate_acc_args(
&self,
batch: &RecordBatch,
) -> Result<EvaluatedAccumulatorArgs> {
- let arguments = self
- .arguments
- .iter()
- .map(|expr| {
- expr.evaluate(batch)
- .and_then(|value| value.into_array(batch.num_rows()))
- })
- .collect::<Result<_>>()?;
-
let filter = self
.filter
.as_ref()
@@ -588,6 +579,19 @@ impl HashAggregateAccumulator {
.and_then(|value| value.into_array(batch.num_rows()))
})
.transpose()?;
+ let selection = filter.as_ref().map(|filter| filter.as_boolean());
+ let arguments = self
+ .arguments
+ .iter()
+ .map(|expr| {
+ selection
+ .map_or_else(
+ || expr.evaluate(batch),
+ |selection| expr.evaluate_selection(batch, selection),
+ )
+ .and_then(|value| value.into_array(batch.num_rows()))
+ })
+ .collect::<Result<_>>()?;
Ok(EvaluatedAccumulatorArgs { arguments, filter })
}
diff --git a/datafusion/sqllogictest/test_files/aggregate_filter_selection.slt b/datafusion/sqllogictest/test_files/aggregate_filter_selection.slt
new file mode 100644
index 0000000000..ca58378584
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/aggregate_filter_selection.slt
@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Aggregate FILTER must select rows before evaluating fallible arguments.
+
+statement ok
+CREATE TABLE aggregate_filter_selection (g INT, v BIGINT);
+
+statement ok
+INSERT INTO aggregate_filter_selection VALUES
+ (1, 0),
+ (2, 2),
+ (3, 0),
+ (4, 5);
+
+statement ok
+SET datafusion.execution.skip_partial_aggregation_probe_rows_threshold = 100000;
+
+statement ok
+SET datafusion.execution.enable_migration_aggregate = true;
+
+query II
+SELECT g, SUM(10 / v) FILTER (WHERE v <> 0)
+FROM aggregate_filter_selection
+GROUP BY g
+ORDER BY g;
+----
+1 NULL
+2 5
+3 NULL
+4 2
+
+statement ok
+SET datafusion.execution.target_partitions = 2;
+
+statement ok
+SET datafusion.execution.batch_size = 1;
+
+statement ok
+SET datafusion.execution.skip_partial_aggregation_probe_rows_threshold = 0;
+
+statement ok
+SET datafusion.execution.skip_partial_aggregation_probe_ratio_threshold = 0.0;
+
+statement ok
+CREATE TABLE aggregate_filter_selection_skip (g INT, v BIGINT);
+
+statement ok
+INSERT INTO aggregate_filter_selection_skip VALUES
+ (1, 2),
+ (3, 5),
+ (2, 0);
+
+statement ok
+SET datafusion.execution.enable_migration_aggregate = true;
+
+query II
+SELECT g, SUM(10 / v) FILTER (WHERE v <> 0)
+FROM aggregate_filter_selection_skip
+GROUP BY g
+ORDER BY g;
+----
+1 5
+2 NULL
+3 2
+
+statement ok
+RESET datafusion.execution.enable_migration_aggregate;
+
+statement ok
+RESET datafusion.execution.skip_partial_aggregation_probe_ratio_threshold;
+
+statement ok
+RESET datafusion.execution.skip_partial_aggregation_probe_rows_threshold;
+
+statement ok
+RESET datafusion.execution.batch_size;
+
+statement ok
+SET datafusion.execution.target_partitions = 4;
+
+statement ok
+DROP TABLE aggregate_filter_selection_skip;
+
+statement ok
+DROP TABLE aggregate_filter_selection;